blob: 91c528cf74d3e43e9fbecb29a7988a5d058ae455 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Markus Handell8e4197b2022-05-30 15:45:28 +020020#include "api/field_trials_view.h"
Markus Handell9a478b52021-11-18 16:07:01 +010021#include "api/rtp_parameters.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020022#include "api/task_queue/default_task_queue_factory.h"
Markus Handell818e7fb2021-12-30 13:01:33 +010023#include "api/task_queue/task_queue_base.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010024#include "api/task_queue/task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020025#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010026#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010027#include "api/test/mock_video_encoder_factory.h"
Markus Handell8d87c462021-12-16 11:37:16 +010028#include "api/units/data_rate.h"
Markus Handellf5a50792022-06-16 14:25:15 +020029#include "api/units/time_delta.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080030#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020031#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020032#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020033#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010034#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010035#include "api/video_codecs/sdp_video_format.h"
Erik Språnge4589cb2022-04-06 16:44:30 +020036#include "api/video_codecs/video_codec.h"
Elad Alon370f93a2019-06-11 14:57:57 +020037#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020038#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010039#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020040#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010041#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020042#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070043#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080044#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020045#include "media/engine/webrtc_video_engine.h"
philipel09a28482022-05-25 09:47:06 +020046#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010047#include "modules/video_coding/codecs/h264/include/h264.h"
48#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
49#include "modules/video_coding/codecs/vp8/include/vp8.h"
50#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020051#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020052#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020053#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010054#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020055#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010056#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020057#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020058#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080059#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020060#include "rtc_base/synchronization/mutex.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020061#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020062#include "test/encoder_settings.h"
63#include "test/fake_encoder.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010064#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020065#include "test/gmock.h"
66#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010067#include "test/mappable_native_buffer.h"
Jonas Orelandc7f691a2022-03-09 15:12:07 +010068#include "test/scoped_key_value_config.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020069#include "test/time_controller/simulated_time_controller.h"
Byoungchan Lee13fe3672022-04-06 10:44:42 +090070#include "test/video_encoder_nullable_proxy_factory.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020071#include "test/video_encoder_proxy_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010072#include "video/frame_cadence_adapter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020073#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070074
75namespace webrtc {
76
sprang57c2fff2017-01-16 06:24:02 -080077using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020078using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020079using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020080using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020081using ::testing::Ge;
82using ::testing::Gt;
Markus Handellee225432021-11-29 12:35:12 +010083using ::testing::Invoke;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020084using ::testing::Le;
85using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010086using ::testing::Matcher;
Markus Handellb4e96d42021-11-05 12:00:55 +010087using ::testing::Mock;
philipel9b058032020-02-10 11:30:00 +010088using ::testing::NiceMock;
Markus Handellb4e96d42021-11-05 12:00:55 +010089using ::testing::Optional;
philipel9b058032020-02-10 11:30:00 +010090using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010091using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020092using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080093
perkj803d97f2016-11-01 11:45:46 -070094namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020095const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010096const int kQpLow = 1;
97const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020098const int kMinFramerateFps = 2;
99const int kMinBalancedFramerateFps = 7;
100const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -0800101const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 10:07:11 +0200102const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
103const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
104const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
105const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 07:02:22 -0800106const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -0700107const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200108const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +0200109const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200110const VideoEncoder::ResolutionBitrateLimits
111 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
112const VideoEncoder::ResolutionBitrateLimits
113 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800114
Asa Persson606d3cb2021-10-04 10:07:11 +0200115uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200116 0x00, 0x00, 0x03, 0x03, 0xF4,
117 0x05, 0x03, 0xC7, 0xE0, 0x1B,
118 0x41, 0x10, 0x8D, 0x00};
119
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100120const uint8_t kCodedFrameVp8Qp25[] = {
121 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
122 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
123 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
124
Markus Handell818e7fb2021-12-30 13:01:33 +0100125VideoFrame CreateSimpleNV12Frame() {
126 return VideoFrame::Builder()
127 .set_video_frame_buffer(rtc::make_ref_counted<NV12Buffer>(
128 /*width=*/16, /*height=*/16))
129 .build();
130}
131
Markus Handell8d87c462021-12-16 11:37:16 +0100132void PassAFrame(
133 TaskQueueBase* encoder_queue,
134 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback,
135 int64_t ntp_time_ms) {
Danil Chapovalov95eeaa72022-07-06 10:14:29 +0200136 encoder_queue->PostTask([video_stream_encoder_callback, ntp_time_ms] {
137 video_stream_encoder_callback->OnFrame(Timestamp::Millis(ntp_time_ms), 1,
138 CreateSimpleNV12Frame());
139 });
Markus Handell8d87c462021-12-16 11:37:16 +0100140}
141
perkj803d97f2016-11-01 11:45:46 -0700142class TestBuffer : public webrtc::I420Buffer {
143 public:
144 TestBuffer(rtc::Event* event, int width, int height)
145 : I420Buffer(width, height), event_(event) {}
146
147 private:
148 friend class rtc::RefCountedObject<TestBuffer>;
149 ~TestBuffer() override {
150 if (event_)
151 event_->Set();
152 }
153 rtc::Event* const event_;
154};
155
Henrik Boström56db9ff2021-03-24 09:06:45 +0100156// A fake native buffer that can't be converted to I420. Upon scaling, it
157// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700158class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
159 public:
160 FakeNativeBuffer(rtc::Event* event, int width, int height)
161 : event_(event), width_(width), height_(height) {}
162 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
163 int width() const override { return width_; }
164 int height() const override { return height_; }
165 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
166 return nullptr;
167 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100168 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
169 int offset_x,
170 int offset_y,
171 int crop_width,
172 int crop_height,
173 int scaled_width,
174 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200175 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
176 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100177 }
Noah Richards51db4212019-06-12 06:59:12 -0700178
179 private:
180 friend class rtc::RefCountedObject<FakeNativeBuffer>;
181 ~FakeNativeBuffer() override {
182 if (event_)
183 event_->Set();
184 }
185 rtc::Event* const event_;
186 const int width_;
187 const int height_;
188};
189
Evan Shrubsole895556e2020-10-05 09:15:13 +0200190// A fake native buffer that is backed by an NV12 buffer.
191class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
192 public:
193 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
194 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
195
196 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
197 int width() const override { return nv12_buffer_->width(); }
198 int height() const override { return nv12_buffer_->height(); }
199 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
200 return nv12_buffer_->ToI420();
201 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200202 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
203 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
204 if (absl::c_find(types, Type::kNV12) != types.end()) {
205 return nv12_buffer_;
206 }
207 return nullptr;
208 }
Niels Möllerba2de582022-04-20 16:46:26 +0200209 const NV12BufferInterface* GetNV12() const { return nv12_buffer_.get(); }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200210
211 private:
212 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
213 ~FakeNV12NativeBuffer() override {
214 if (event_)
215 event_->Set();
216 }
217 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
218 rtc::Event* const event_;
219};
220
Niels Möller7dc26b72017-12-06 10:27:48 +0100221class CpuOveruseDetectorProxy : public OveruseFrameDetector {
222 public:
Markus Handell8e4197b2022-05-30 15:45:28 +0200223 CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer,
224 const FieldTrialsView& field_trials)
225 : OveruseFrameDetector(metrics_observer, field_trials),
Henrik Boström381d1092020-05-12 18:49:07 +0200226 last_target_framerate_fps_(-1),
227 framerate_updated_event_(true /* manual_reset */,
228 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100229 virtual ~CpuOveruseDetectorProxy() {}
230
231 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200232 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100233 last_target_framerate_fps_ = framerate_fps;
234 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200235 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100236 }
237
238 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200239 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100240 return last_target_framerate_fps_;
241 }
242
Niels Möller4db138e2018-04-19 09:04:13 +0200243 CpuOveruseOptions GetOptions() { return options_; }
244
Henrik Boström381d1092020-05-12 18:49:07 +0200245 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
246
Niels Möller7dc26b72017-12-06 10:27:48 +0100247 private:
Markus Handella3765182020-07-08 13:13:32 +0200248 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100249 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200250 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100251};
252
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200253class FakeVideoSourceRestrictionsListener
254 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200255 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200256 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200257 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200258 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200259 RTC_DCHECK(was_restrictions_updated_);
260 }
261
262 rtc::Event* restrictions_updated_event() {
263 return &restrictions_updated_event_;
264 }
265
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200266 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200267 void OnVideoSourceRestrictionsUpdated(
268 VideoSourceRestrictions restrictions,
269 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200270 rtc::scoped_refptr<Resource> reason,
271 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200272 was_restrictions_updated_ = true;
273 restrictions_updated_event_.Set();
274 }
275
276 private:
277 bool was_restrictions_updated_;
278 rtc::Event restrictions_updated_event_;
279};
280
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200281auto WantsFps(Matcher<int> fps_matcher) {
282 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
283 fps_matcher);
284}
285
286auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
287 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
288 AllOf(max_pixel_matcher, Gt(0)));
289}
290
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200291auto ResolutionMax() {
292 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200293 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200294 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
295 Eq(absl::nullopt)));
296}
297
298auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200299 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200300}
301
302auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200303 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200304}
305
306auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200307 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200308}
309
310auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200311 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200312}
313
314auto FpsMaxResolutionMax() {
315 return AllOf(FpsMax(), ResolutionMax());
316}
317
318auto UnlimitedSinkWants() {
319 return AllOf(FpsUnlimited(), ResolutionMax());
320}
321
322auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
323 Matcher<int> fps_range_matcher;
324
325 if (last_frame_pixels <= 320 * 240) {
326 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200327 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200328 fps_range_matcher = AllOf(Ge(10), Le(15));
329 } else if (last_frame_pixels <= 640 * 480) {
330 fps_range_matcher = Ge(15);
331 } else {
332 fps_range_matcher = Eq(kDefaultFramerate);
333 }
334 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
335 fps_range_matcher);
336}
337
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200338auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
339 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
340 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
341}
342
343auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
344 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
345}
346
347auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
348 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
349}
350
351auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
352 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
353 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
354}
355
356auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
357 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
358 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
359}
360
361auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
362 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
363 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
364}
365
366auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
367 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
368 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
369}
370
mflodmancc3d4422017-08-03 08:27:51 -0700371class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700372 public:
Markus Handell9a478b52021-11-18 16:07:01 +0100373 VideoStreamEncoderUnderTest(
374 TimeController* time_controller,
375 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter,
376 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
377 encoder_queue,
378 SendStatisticsProxy* stats_proxy,
379 const VideoStreamEncoderSettings& settings,
380 VideoStreamEncoder::BitrateAllocationCallbackType
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100381 allocation_callback_type,
Erik Språnge4589cb2022-04-06 16:44:30 +0200382 const FieldTrialsView& field_trials,
383 int num_cores)
Markus Handell8e4197b2022-05-30 15:45:28 +0200384 : VideoStreamEncoder(
385 time_controller->GetClock(),
386 num_cores,
387 stats_proxy,
388 settings,
389 std::unique_ptr<OveruseFrameDetector>(
390 overuse_detector_proxy_ =
391 new CpuOveruseDetectorProxy(stats_proxy, field_trials)),
392 std::move(cadence_adapter),
393 std::move(encoder_queue),
394 allocation_callback_type,
395 field_trials),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200396 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200397 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200398 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200399 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200400 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200401 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200402 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200403 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100404 }
perkj803d97f2016-11-01 11:45:46 -0700405
Henrik Boström381d1092020-05-12 18:49:07 +0200406 void SetSourceAndWaitForRestrictionsUpdated(
407 rtc::VideoSourceInterface<VideoFrame>* source,
408 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200409 FakeVideoSourceRestrictionsListener listener;
410 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200411 SetSource(source, degradation_preference);
412 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200413 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200414 }
415
416 void SetSourceAndWaitForFramerateUpdated(
417 rtc::VideoSourceInterface<VideoFrame>* source,
418 const DegradationPreference& degradation_preference) {
419 overuse_detector_proxy_->framerate_updated_event()->Reset();
420 SetSource(source, degradation_preference);
421 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
422 }
423
424 void OnBitrateUpdatedAndWaitForManagedResources(
425 DataRate target_bitrate,
426 DataRate stable_target_bitrate,
427 DataRate link_allocation,
428 uint8_t fraction_lost,
429 int64_t round_trip_time_ms,
430 double cwnd_reduce_ratio) {
431 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
432 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
433 // Bitrate is updated on the encoder queue.
434 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200435 }
436
kthelgason2fc52542017-03-03 00:24:41 -0800437 // This is used as a synchronisation mechanism, to make sure that the
438 // encoder queue is not blocked before we start sending it frames.
439 void WaitUntilTaskQueueIsIdle() {
Markus Handell28c71802021-11-08 10:11:55 +0100440 time_controller_->AdvanceTime(TimeDelta::Zero());
kthelgason2fc52542017-03-03 00:24:41 -0800441 }
442
Henrik Boström91aa7322020-04-28 12:24:33 +0200443 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200444 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200445 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200446 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200447 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200448 event.Set();
449 });
450 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100451 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200452 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200453
Henrik Boström91aa7322020-04-28 12:24:33 +0200454 void TriggerCpuUnderuse() {
455 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200456 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200457 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200458 event.Set();
459 });
460 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100461 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200462 }
kthelgason876222f2016-11-29 01:44:11 -0800463
Henrik Boström91aa7322020-04-28 12:24:33 +0200464 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200465 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200466 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200467 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200468 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200469 event.Set();
470 });
471 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100472 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200473 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200474 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200475 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200476 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200477 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200478 event.Set();
479 });
480 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100481 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200482 }
483
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200484 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100485 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200486 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
487 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200488 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700489};
490
Noah Richards51db4212019-06-12 06:59:12 -0700491// Simulates simulcast behavior and makes highest stream resolutions divisible
492// by 4.
493class CroppingVideoStreamFactory
494 : public VideoEncoderConfig::VideoStreamFactoryInterface {
495 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200496 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700497
498 private:
499 std::vector<VideoStream> CreateEncoderStreams(
500 int width,
501 int height,
502 const VideoEncoderConfig& encoder_config) override {
503 std::vector<VideoStream> streams = test::CreateVideoStreams(
504 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700505 return streams;
506 }
Noah Richards51db4212019-06-12 06:59:12 -0700507};
508
sprangb1ca0732017-02-01 08:38:12 -0800509class AdaptingFrameForwarder : public test::FrameForwarder {
510 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200511 explicit AdaptingFrameForwarder(TimeController* time_controller)
512 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700513 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800514
515 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200516 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800517 adaptation_enabled_ = enabled;
518 }
519
asaperssonfab67072017-04-04 05:51:49 -0700520 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200521 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800522 return adaptation_enabled_;
523 }
524
Henrik Boström1124ed12021-02-25 10:30:39 +0100525 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
526 // the resolution or frame rate was different than it is currently. If
527 // something else is modified, such as encoder resolutions, but the resolution
528 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700529 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200530 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700531 return last_wants_;
532 }
533
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200534 absl::optional<int> last_sent_width() const { return last_width_; }
535 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800536
sprangb1ca0732017-02-01 08:38:12 -0800537 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200538 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +0100539 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200540
sprangb1ca0732017-02-01 08:38:12 -0800541 int cropped_width = 0;
542 int cropped_height = 0;
543 int out_width = 0;
544 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700545 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000546 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
547 << "w=" << video_frame.width()
548 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700549 if (adapter_.AdaptFrameResolution(
550 video_frame.width(), video_frame.height(),
551 video_frame.timestamp_us() * 1000, &cropped_width,
552 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100553 VideoFrame adapted_frame =
554 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200555 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100556 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200557 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100558 .set_timestamp_ms(99)
559 .set_rotation(kVideoRotation_0)
560 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100561 if (video_frame.has_update_rect()) {
562 adapted_frame.set_update_rect(
563 video_frame.update_rect().ScaleWithFrame(
564 video_frame.width(), video_frame.height(), 0, 0,
565 video_frame.width(), video_frame.height(), out_width,
566 out_height));
567 }
sprangc5d62e22017-04-02 23:53:04 -0700568 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800569 last_width_.emplace(adapted_frame.width());
570 last_height_.emplace(adapted_frame.height());
571 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200572 last_width_ = absl::nullopt;
573 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700574 }
sprangb1ca0732017-02-01 08:38:12 -0800575 } else {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000576 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800577 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800578 last_width_.emplace(video_frame.width());
579 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800580 }
581 }
582
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200583 void OnOutputFormatRequest(int width, int height) {
584 absl::optional<std::pair<int, int>> target_aspect_ratio =
585 std::make_pair(width, height);
586 absl::optional<int> max_pixel_count = width * height;
587 absl::optional<int> max_fps;
588 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
589 max_fps);
590 }
591
sprangb1ca0732017-02-01 08:38:12 -0800592 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
593 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200594 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100595 rtc::VideoSinkWants prev_wants = sink_wants_locked();
596 bool did_adapt =
597 prev_wants.max_pixel_count != wants.max_pixel_count ||
598 prev_wants.target_pixel_count != wants.target_pixel_count ||
599 prev_wants.max_framerate_fps != wants.max_framerate_fps;
600 if (did_adapt) {
601 last_wants_ = prev_wants;
602 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100603 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200604 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800605 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200606
607 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800608 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200609 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
610 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200611 absl::optional<int> last_width_;
612 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800613};
sprangc5d62e22017-04-02 23:53:04 -0700614
Niels Möller213618e2018-07-24 09:29:58 +0200615// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700616class MockableSendStatisticsProxy : public SendStatisticsProxy {
617 public:
618 MockableSendStatisticsProxy(Clock* clock,
619 const VideoSendStream::Config& config,
Jonas Oreland8ca06132022-03-14 12:52:48 +0100620 VideoEncoderConfig::ContentType content_type,
Jonas Orelande62c2f22022-03-29 11:04:48 +0200621 const FieldTrialsView& field_trials)
Jonas Oreland8ca06132022-03-14 12:52:48 +0100622 : SendStatisticsProxy(clock, config, content_type, field_trials) {}
sprangc5d62e22017-04-02 23:53:04 -0700623
624 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200625 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700626 if (mock_stats_)
627 return *mock_stats_;
628 return SendStatisticsProxy::GetStats();
629 }
630
Niels Möller213618e2018-07-24 09:29:58 +0200631 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200632 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200633 if (mock_stats_)
634 return mock_stats_->input_frame_rate;
635 return SendStatisticsProxy::GetInputFrameRate();
636 }
sprangc5d62e22017-04-02 23:53:04 -0700637 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200638 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700639 mock_stats_.emplace(stats);
640 }
641
642 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200643 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700644 mock_stats_.reset();
645 }
646
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200647 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
648 on_frame_dropped_ = std::move(callback);
649 }
650
sprangc5d62e22017-04-02 23:53:04 -0700651 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200652 void OnFrameDropped(DropReason reason) override {
653 SendStatisticsProxy::OnFrameDropped(reason);
654 if (on_frame_dropped_)
655 on_frame_dropped_(reason);
656 }
657
Markus Handella3765182020-07-08 13:13:32 +0200658 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200659 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200660 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700661};
662
Markus Handellb4e96d42021-11-05 12:00:55 +0100663class SimpleVideoStreamEncoderFactory {
664 public:
665 class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
666 public:
667 using VideoStreamEncoder::VideoStreamEncoder;
668 ~AdaptedVideoStreamEncoder() { Stop(); }
669 };
670
Markus Handell8d87c462021-12-16 11:37:16 +0100671 class MockFakeEncoder : public test::FakeEncoder {
672 public:
673 using FakeEncoder::FakeEncoder;
674 MOCK_METHOD(CodecSpecificInfo,
675 EncodeHook,
676 (EncodedImage & encoded_image,
677 rtc::scoped_refptr<EncodedImageBuffer> buffer),
678 (override));
679 };
680
Markus Handellee225432021-11-29 12:35:12 +0100681 SimpleVideoStreamEncoderFactory() {
Markus Handellb4e96d42021-11-05 12:00:55 +0100682 encoder_settings_.encoder_factory = &encoder_factory_;
Markus Handellee225432021-11-29 12:35:12 +0100683 encoder_settings_.bitrate_allocator_factory =
684 bitrate_allocator_factory_.get();
Markus Handellb4e96d42021-11-05 12:00:55 +0100685 }
686
Markus Handell818e7fb2021-12-30 13:01:33 +0100687 std::unique_ptr<AdaptedVideoStreamEncoder> CreateWithEncoderQueue(
Markus Handellee225432021-11-29 12:35:12 +0100688 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100689 std::unique_ptr<TaskQueueBase, TaskQueueDeleter> encoder_queue,
Jonas Orelande62c2f22022-03-29 11:04:48 +0200690 const FieldTrialsView* field_trials = nullptr) {
Markus Handellb4e96d42021-11-05 12:00:55 +0100691 auto result = std::make_unique<AdaptedVideoStreamEncoder>(
692 time_controller_.GetClock(),
693 /*number_of_cores=*/1,
694 /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
Markus Handell8e4197b2022-05-30 15:45:28 +0200695 std::make_unique<CpuOveruseDetectorProxy>(
696 /*stats_proxy=*/nullptr,
697 field_trials ? *field_trials : field_trials_),
Markus Handellee225432021-11-29 12:35:12 +0100698 std::move(zero_hertz_adapter), std::move(encoder_queue),
Markus Handellb4e96d42021-11-05 12:00:55 +0100699 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100700 kVideoBitrateAllocation,
701 field_trials ? *field_trials : field_trials_);
Markus Handellb4e96d42021-11-05 12:00:55 +0100702 result->SetSink(&sink_, /*rotation_applied=*/false);
703 return result;
704 }
705
Markus Handell818e7fb2021-12-30 13:01:33 +0100706 std::unique_ptr<AdaptedVideoStreamEncoder> Create(
707 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
708 TaskQueueBase** encoder_queue_ptr = nullptr) {
709 auto encoder_queue =
710 time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
711 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
712 if (encoder_queue_ptr)
713 *encoder_queue_ptr = encoder_queue.get();
714 return CreateWithEncoderQueue(std::move(zero_hertz_adapter),
715 std::move(encoder_queue));
716 }
717
Markus Handell9a478b52021-11-18 16:07:01 +0100718 void DepleteTaskQueues() { time_controller_.AdvanceTime(TimeDelta::Zero()); }
Markus Handell8d87c462021-12-16 11:37:16 +0100719 MockFakeEncoder& GetMockFakeEncoder() { return mock_fake_encoder_; }
Markus Handell9a478b52021-11-18 16:07:01 +0100720
Markus Handell818e7fb2021-12-30 13:01:33 +0100721 GlobalSimulatedTimeController* GetTimeController() {
722 return &time_controller_;
723 }
724
Markus Handellb4e96d42021-11-05 12:00:55 +0100725 private:
726 class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
727 public:
728 ~NullEncoderSink() override = default;
729 void OnEncoderConfigurationChanged(
730 std::vector<VideoStream> streams,
731 bool is_svc,
732 VideoEncoderConfig::ContentType content_type,
733 int min_transmit_bitrate_bps) override {}
734 void OnBitrateAllocationUpdated(
735 const VideoBitrateAllocation& allocation) override {}
736 void OnVideoLayersAllocationUpdated(
737 VideoLayersAllocation allocation) override {}
738 Result OnEncodedImage(
739 const EncodedImage& encoded_image,
740 const CodecSpecificInfo* codec_specific_info) override {
741 return Result(EncodedImageCallback::Result::OK);
742 }
743 };
744
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100745 test::ScopedKeyValueConfig field_trials_;
Markus Handellee225432021-11-29 12:35:12 +0100746 GlobalSimulatedTimeController time_controller_{Timestamp::Millis(0)};
747 std::unique_ptr<TaskQueueFactory> task_queue_factory_{
748 time_controller_.CreateTaskQueueFactory()};
749 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_ =
750 std::make_unique<MockableSendStatisticsProxy>(
751 time_controller_.GetClock(),
752 VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 12:52:48 +0100753 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
754 field_trials_);
Markus Handellee225432021-11-29 12:35:12 +0100755 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_ =
756 CreateBuiltinVideoBitrateAllocatorFactory();
757 VideoStreamEncoderSettings encoder_settings_{
758 VideoEncoder::Capabilities(/*loss_notification=*/false)};
Markus Handell8d87c462021-12-16 11:37:16 +0100759 MockFakeEncoder mock_fake_encoder_{time_controller_.GetClock()};
760 test::VideoEncoderProxyFactory encoder_factory_{&mock_fake_encoder_};
Markus Handellb4e96d42021-11-05 12:00:55 +0100761 NullEncoderSink sink_;
762};
763
764class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
765 public:
766 MOCK_METHOD(void, Initialize, (Callback * callback), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100767 MOCK_METHOD(void,
768 SetZeroHertzModeEnabled,
769 (absl::optional<ZeroHertzModeParams>),
770 (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100771 MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
Markus Handellee225432021-11-29 12:35:12 +0100772 MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override));
773 MOCK_METHOD(void, UpdateFrameRate, (), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100774 MOCK_METHOD(void,
775 UpdateLayerQualityConvergence,
776 (int spatial_index, bool converged),
777 (override));
778 MOCK_METHOD(void,
779 UpdateLayerStatus,
780 (int spatial_index, bool enabled),
781 (override));
Markus Handell818e7fb2021-12-30 13:01:33 +0100782 MOCK_METHOD(void, ProcessKeyFrameRequest, (), (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100783};
784
philipel9b058032020-02-10 11:30:00 +0100785class MockEncoderSelector
786 : public VideoEncoderFactory::EncoderSelectorInterface {
787 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200788 MOCK_METHOD(void,
789 OnCurrentEncoder,
790 (const SdpVideoFormat& format),
791 (override));
792 MOCK_METHOD(absl::optional<SdpVideoFormat>,
793 OnAvailableBitrate,
794 (const DataRate& rate),
795 (override));
philipel6daa3042022-04-11 10:48:28 +0200796 MOCK_METHOD(absl::optional<SdpVideoFormat>,
797 OnResolutionChange,
798 (const RenderResolution& resolution),
799 (override));
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200800 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100801};
802
Markus Handell2e0f4f02021-12-21 19:14:58 +0100803class MockVideoSourceInterface : public rtc::VideoSourceInterface<VideoFrame> {
804 public:
805 MOCK_METHOD(void,
806 AddOrUpdateSink,
807 (rtc::VideoSinkInterface<VideoFrame>*,
808 const rtc::VideoSinkWants&),
809 (override));
810 MOCK_METHOD(void,
811 RemoveSink,
812 (rtc::VideoSinkInterface<VideoFrame>*),
813 (override));
814 MOCK_METHOD(void, RequestRefreshFrame, (), (override));
815};
816
perkj803d97f2016-11-01 11:45:46 -0700817} // namespace
818
mflodmancc3d4422017-08-03 08:27:51 -0700819class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700820 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200821 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700822
mflodmancc3d4422017-08-03 08:27:51 -0700823 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700824 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700825 codec_width_(320),
826 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200827 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200828 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200829 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700830 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200831 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700832 video_send_config_,
Jonas Oreland8ca06132022-03-14 12:52:48 +0100833 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
834 field_trials_)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200835 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700836
837 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700838 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700839 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200840 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800841 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200842 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200843 video_send_config_.rtp.payload_name = "FAKE";
844 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700845
Per512ecb32016-09-23 15:52:06 +0200846 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200847 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200848 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
849 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
850 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100851 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700852
Niels Möllerf1338562018-04-26 09:51:47 +0200853 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800854 }
855
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100856 void ConfigureEncoder(
857 VideoEncoderConfig video_encoder_config,
858 VideoStreamEncoder::BitrateAllocationCallbackType
859 allocation_callback_type =
860 VideoStreamEncoder::BitrateAllocationCallbackType::
Erik Språnge4589cb2022-04-06 16:44:30 +0200861 kVideoBitrateAllocationWhenScreenSharing,
862 int num_cores = 1) {
mflodmancc3d4422017-08-03 08:27:51 -0700863 if (video_stream_encoder_)
864 video_stream_encoder_->Stop();
Markus Handell9a478b52021-11-18 16:07:01 +0100865
866 auto encoder_queue = GetTaskQueueFactory()->CreateTaskQueue(
867 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
868 TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
869 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter =
870 FrameCadenceAdapterInterface::Create(time_controller_.GetClock(),
Jonas Oreland8ca06132022-03-14 12:52:48 +0100871 encoder_queue_ptr, field_trials_);
Markus Handell9a478b52021-11-18 16:07:01 +0100872 video_stream_encoder_ = std::make_unique<VideoStreamEncoderUnderTest>(
873 &time_controller_, std::move(cadence_adapter), std::move(encoder_queue),
874 stats_proxy_.get(), video_send_config_.encoder_settings,
Erik Språnge4589cb2022-04-06 16:44:30 +0200875 allocation_callback_type, field_trials_, num_cores);
Asa Persson606d3cb2021-10-04 10:07:11 +0200876 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 08:27:51 -0700877 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700878 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 10:07:11 +0200879 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 08:27:51 -0700880 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200881 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700882 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800883 }
884
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100885 void ResetEncoder(const std::string& payload_name,
886 size_t num_streams,
887 size_t num_temporal_layers,
888 unsigned char num_spatial_layers,
889 bool screenshare,
890 VideoStreamEncoder::BitrateAllocationCallbackType
891 allocation_callback_type =
892 VideoStreamEncoder::BitrateAllocationCallbackType::
Erik Språnge4589cb2022-04-06 16:44:30 +0200893 kVideoBitrateAllocationWhenScreenSharing,
894 int num_cores = 1) {
Niels Möller259a4972018-04-05 15:36:51 +0200895 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800896
897 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200898 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
899 num_streams, &video_encoder_config);
900 for (auto& layer : video_encoder_config.simulcast_layers) {
901 layer.num_temporal_layers = num_temporal_layers;
902 layer.max_framerate = kDefaultFramerate;
903 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100904 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 10:07:11 +0200905 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 07:06:52 -0700906 video_encoder_config.content_type =
907 screenshare ? VideoEncoderConfig::ContentType::kScreen
908 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700909 if (payload_name == "VP9") {
910 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
911 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200912 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700913 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200914 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
915 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700916 }
Erik Språnge4589cb2022-04-06 16:44:30 +0200917 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type,
918 num_cores);
perkj26091b12016-09-01 01:17:40 -0700919 }
920
sprang57c2fff2017-01-16 06:24:02 -0800921 VideoFrame CreateFrame(int64_t ntp_time_ms,
922 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200923 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200924 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200925 destruction_event, codec_width_, codec_height_))
926 .set_ntp_time_ms(ntp_time_ms)
927 .set_timestamp_ms(99)
928 .set_rotation(kVideoRotation_0)
929 .build();
perkj26091b12016-09-01 01:17:40 -0700930 }
931
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100932 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
933 rtc::Event* destruction_event,
934 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200935 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200936 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200937 destruction_event, codec_width_, codec_height_))
938 .set_ntp_time_ms(ntp_time_ms)
939 .set_timestamp_ms(99)
940 .set_rotation(kVideoRotation_0)
941 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
942 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100943 }
944
sprang57c2fff2017-01-16 06:24:02 -0800945 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200946 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
947 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200948 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200949 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200950 .set_ntp_time_ms(ntp_time_ms)
951 .set_timestamp_ms(ntp_time_ms)
952 .set_rotation(kVideoRotation_0)
953 .build();
perkj803d97f2016-11-01 11:45:46 -0700954 }
955
Evan Shrubsole895556e2020-10-05 09:15:13 +0200956 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200957 return VideoFrame::Builder()
958 .set_video_frame_buffer(NV12Buffer::Create(width, height))
959 .set_ntp_time_ms(ntp_time_ms)
960 .set_timestamp_ms(ntp_time_ms)
961 .set_rotation(kVideoRotation_0)
962 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200963 }
964
Noah Richards51db4212019-06-12 06:59:12 -0700965 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
966 rtc::Event* destruction_event,
967 int width,
968 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200969 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200970 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200971 destruction_event, width, height))
972 .set_ntp_time_ms(ntp_time_ms)
973 .set_timestamp_ms(99)
974 .set_rotation(kVideoRotation_0)
975 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700976 }
977
Evan Shrubsole895556e2020-10-05 09:15:13 +0200978 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
979 rtc::Event* destruction_event,
980 int width,
981 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200982 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200983 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200984 destruction_event, width, height))
985 .set_ntp_time_ms(ntp_time_ms)
986 .set_timestamp_ms(99)
987 .set_rotation(kVideoRotation_0)
988 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200989 }
990
Noah Richards51db4212019-06-12 06:59:12 -0700991 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
992 rtc::Event* destruction_event) const {
993 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
994 codec_height_);
995 }
996
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100997 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200998 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +0200999 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001000
1001 video_source_.IncomingCapturedFrame(
1002 CreateFrame(1, codec_width_, codec_height_));
1003 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +02001004 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001005 }
1006
sprang4847ae62017-06-27 07:06:52 -07001007 void WaitForEncodedFrame(int64_t expected_ntp_time) {
1008 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001009 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001010 }
1011
1012 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
1013 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001014 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001015 return ok;
1016 }
1017
1018 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
1019 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001020 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001021 }
1022
1023 void ExpectDroppedFrame() {
1024 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001025 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001026 }
1027
1028 bool WaitForFrame(int64_t timeout_ms) {
1029 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001030 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001031 return ok;
1032 }
1033
perkj26091b12016-09-01 01:17:40 -07001034 class TestEncoder : public test::FakeEncoder {
1035 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001036 explicit TestEncoder(TimeController* time_controller)
1037 : FakeEncoder(time_controller->GetClock()),
1038 time_controller_(time_controller) {
1039 RTC_DCHECK(time_controller_);
1040 }
perkj26091b12016-09-01 01:17:40 -07001041
Erik Språngaed30702018-11-05 12:57:17 +01001042 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +02001043 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +02001044 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +01001045 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001046 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +01001047 info.scaling_settings = VideoEncoder::ScalingSettings(
1048 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001049 }
1050 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001051 for (int i = 0; i < kMaxSpatialLayers; ++i) {
1052 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001053 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001054 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001055 for (int tid = 0; tid < num_layers; ++tid)
1056 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001057 }
1058 }
Erik Språngaed30702018-11-05 12:57:17 +01001059 }
Sergey Silkin6456e352019-07-08 17:56:40 +02001060
1061 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001062 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001063 info.apply_alignment_to_all_simulcast_layers =
1064 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001065 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001066 if (is_qp_trusted_.has_value()) {
1067 info.is_qp_trusted = is_qp_trusted_;
1068 }
Erik Språngaed30702018-11-05 12:57:17 +01001069 return info;
kthelgason876222f2016-11-29 01:44:11 -08001070 }
1071
Erik Språngb7cb7b52019-02-26 15:52:33 +01001072 int32_t RegisterEncodeCompleteCallback(
1073 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +02001074 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001075 encoded_image_callback_ = callback;
1076 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
1077 }
1078
perkjfa10b552016-10-02 23:45:26 -07001079 void ContinueEncode() { continue_encode_event_.Set(); }
1080
1081 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
1082 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +02001083 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -07001084 EXPECT_EQ(timestamp_, timestamp);
1085 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
1086 }
1087
kthelgason2fc52542017-03-03 00:24:41 -08001088 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +02001089 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -08001090 quality_scaling_ = b;
1091 }
kthelgasonad9010c2017-02-14 00:46:51 -08001092
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001093 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +02001094 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001095 requested_resolution_alignment_ = requested_resolution_alignment;
1096 }
1097
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001098 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
1099 MutexLock lock(&local_mutex_);
1100 apply_alignment_to_all_simulcast_layers_ = b;
1101 }
1102
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001103 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +02001104 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001105 is_hardware_accelerated_ = is_hardware_accelerated;
1106 }
1107
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001108 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
1109 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +02001110 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001111 temporal_layers_supported_[spatial_idx] = supported;
1112 }
1113
Sergey Silkin6456e352019-07-08 17:56:40 +02001114 void SetResolutionBitrateLimits(
1115 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +02001116 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +02001117 resolution_bitrate_limits_ = thresholds;
1118 }
1119
sprangfe627f32017-03-29 08:24:59 -07001120 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +02001121 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -07001122 force_init_encode_failed_ = force_failure;
1123 }
1124
Niels Möller6bb5ab92019-01-11 11:11:10 +01001125 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +02001126 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001127 rate_factor_ = rate_factor;
1128 }
1129
Erik Språngd7329ca2019-02-21 21:19:53 +01001130 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +02001131 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001132 return last_framerate_;
1133 }
1134
Erik Språngd7329ca2019-02-21 21:19:53 +01001135 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +02001136 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001137 return last_update_rect_;
1138 }
1139
Niels Möller87e2d782019-03-07 10:18:23 +01001140 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +02001141 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001142 return last_frame_types_;
1143 }
1144
1145 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +01001146 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +01001147 keyframe ? VideoFrameType::kVideoFrameKey
1148 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01001149 {
Markus Handella3765182020-07-08 13:13:32 +02001150 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001151 last_frame_types_ = frame_type;
1152 }
Niels Möllerb859b322019-03-07 12:40:01 +01001153 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +01001154 }
1155
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001156 void InjectEncodedImage(const EncodedImage& image,
1157 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +02001158 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001159 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001160 }
1161
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001162 void SetEncodedImageData(
1163 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +02001164 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001165 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001166 }
1167
Erik Språngd7329ca2019-02-21 21:19:53 +01001168 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +02001169 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001170 expect_null_frame_ = true;
1171 }
1172
Erik Språng5056af02019-09-02 15:53:11 +02001173 absl::optional<VideoEncoder::RateControlParameters>
1174 GetAndResetLastRateControlSettings() {
1175 auto settings = last_rate_control_settings_;
1176 last_rate_control_settings_.reset();
1177 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +01001178 }
1179
Henrik Boström56db9ff2021-03-24 09:06:45 +01001180 int GetLastInputWidth() const {
1181 MutexLock lock(&local_mutex_);
1182 return last_input_width_;
1183 }
1184
1185 int GetLastInputHeight() const {
1186 MutexLock lock(&local_mutex_);
1187 return last_input_height_;
1188 }
1189
Evan Shrubsole895556e2020-10-05 09:15:13 +02001190 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1191 MutexLock lock(&local_mutex_);
1192 return last_input_pixel_format_;
1193 }
1194
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001195 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001196 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001197 return num_set_rates_;
1198 }
1199
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001200 void SetPreferredPixelFormats(
1201 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1202 pixel_formats) {
1203 MutexLock lock(&local_mutex_);
1204 preferred_pixel_formats_ = std::move(pixel_formats);
1205 }
1206
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001207 void SetIsQpTrusted(absl::optional<bool> trusted) {
1208 MutexLock lock(&local_mutex_);
1209 is_qp_trusted_ = trusted;
1210 }
1211
Erik Språnge4589cb2022-04-06 16:44:30 +02001212 VideoCodecComplexity LastEncoderComplexity() {
1213 MutexLock lock(&local_mutex_);
1214 return last_encoder_complexity_;
1215 }
1216
perkjfa10b552016-10-02 23:45:26 -07001217 private:
perkj26091b12016-09-01 01:17:40 -07001218 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001219 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001220 {
Markus Handella3765182020-07-08 13:13:32 +02001221 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001222 if (expect_null_frame_) {
1223 EXPECT_EQ(input_image.timestamp(), 0u);
1224 EXPECT_EQ(input_image.width(), 1);
1225 last_frame_types_ = *frame_types;
1226 expect_null_frame_ = false;
1227 } else {
1228 EXPECT_GT(input_image.timestamp(), timestamp_);
1229 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1230 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1231 }
perkj26091b12016-09-01 01:17:40 -07001232
1233 timestamp_ = input_image.timestamp();
1234 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001235 last_input_width_ = input_image.width();
1236 last_input_height_ = input_image.height();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001237 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001238 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001239 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001240 }
Niels Möllerb859b322019-03-07 12:40:01 +01001241 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001242 return result;
1243 }
1244
Niels Möller08ae7ce2020-09-23 15:58:12 +02001245 CodecSpecificInfo EncodeHook(
1246 EncodedImage& encoded_image,
1247 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001248 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001249 {
1250 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001251 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001252 }
1253 MutexLock lock(&local_mutex_);
1254 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001255 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001256 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001257 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001258 }
1259
sprangfe627f32017-03-29 08:24:59 -07001260 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001261 const Settings& settings) override {
1262 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001263
Markus Handella3765182020-07-08 13:13:32 +02001264 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001265 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001266
Erik Språng82fad3d2018-03-21 09:57:23 +01001267 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001268 // Simulate setting up temporal layers, in order to validate the life
1269 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001270 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001271 frame_buffer_controller_ =
1272 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001273 }
Erik Språnge4589cb2022-04-06 16:44:30 +02001274
1275 last_encoder_complexity_ = config->GetVideoEncoderComplexity();
1276
Erik Språngb7cb7b52019-02-26 15:52:33 +01001277 if (force_init_encode_failed_) {
1278 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001279 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001280 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001281
Erik Språngb7cb7b52019-02-26 15:52:33 +01001282 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001283 return res;
1284 }
1285
Erik Språngb7cb7b52019-02-26 15:52:33 +01001286 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001287 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001288 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1289 initialized_ = EncoderState::kUninitialized;
1290 return FakeEncoder::Release();
1291 }
1292
Erik Språng16cb8f52019-04-12 13:59:09 +02001293 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001294 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001295 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001296 VideoBitrateAllocation adjusted_rate_allocation;
1297 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1298 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001299 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001300 adjusted_rate_allocation.SetBitrate(
1301 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001302 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001303 rate_factor_));
1304 }
1305 }
1306 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001307 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001308 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001309 RateControlParameters adjusted_paramters = parameters;
1310 adjusted_paramters.bitrate = adjusted_rate_allocation;
1311 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001312 }
1313
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001314 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001315 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001316 enum class EncoderState {
1317 kUninitialized,
1318 kInitializationFailed,
1319 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001320 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
perkj26091b12016-09-01 01:17:40 -07001321 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001322 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1323 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1324 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1325 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1326 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1327 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001328 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1329 false;
Markus Handella3765182020-07-08 13:13:32 +02001330 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001331 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1332 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001333 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001334 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001335 absl::optional<bool>
1336 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001337 local_mutex_);
1338 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1339 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1340 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001341 absl::optional<VideoEncoder::RateControlParameters>
1342 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001343 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1344 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001345 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001346 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001347 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1348 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001349 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 17:56:40 +02001350 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001351 RTC_GUARDED_BY(local_mutex_);
1352 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001353 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1354 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001355 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1356 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001357 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
Erik Språnge4589cb2022-04-06 16:44:30 +02001358 VideoCodecComplexity last_encoder_complexity_ RTC_GUARDED_BY(local_mutex_){
1359 VideoCodecComplexity::kComplexityNormal};
perkj26091b12016-09-01 01:17:40 -07001360 };
1361
mflodmancc3d4422017-08-03 08:27:51 -07001362 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001363 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001364 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1365 : time_controller_(time_controller), test_encoder_(test_encoder) {
1366 RTC_DCHECK(time_controller_);
1367 }
perkj26091b12016-09-01 01:17:40 -07001368
perkj26091b12016-09-01 01:17:40 -07001369 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001370 EXPECT_TRUE(
1371 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1372 }
1373
1374 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1375 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001376 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001377 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001378 return false;
perkj26091b12016-09-01 01:17:40 -07001379 {
Markus Handella3765182020-07-08 13:13:32 +02001380 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001381 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001382 }
1383 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001384 return true;
perkj26091b12016-09-01 01:17:40 -07001385 }
1386
sprangb1ca0732017-02-01 08:38:12 -08001387 void WaitForEncodedFrame(uint32_t expected_width,
1388 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001389 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001390 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001391 }
1392
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001393 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001394 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001395 uint32_t width = 0;
1396 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001397 {
Markus Handella3765182020-07-08 13:13:32 +02001398 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001399 width = last_width_;
1400 height = last_height_;
1401 }
1402 EXPECT_EQ(expected_height, height);
1403 EXPECT_EQ(expected_width, width);
1404 }
1405
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001406 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1407 VideoRotation rotation;
1408 {
Markus Handella3765182020-07-08 13:13:32 +02001409 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001410 rotation = last_rotation_;
1411 }
1412 EXPECT_EQ(expected_rotation, rotation);
1413 }
1414
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001415 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001416
sprangc5d62e22017-04-02 23:53:04 -07001417 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001418 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +01001419 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001420 bool ret = encoded_frame_event_.Wait(timeout_ms);
Markus Handell28c71802021-11-08 10:11:55 +01001421 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001422 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001423 }
1424
perkj26091b12016-09-01 01:17:40 -07001425 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001426 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001427 expect_frames_ = false;
1428 }
1429
asaperssonfab67072017-04-04 05:51:49 -07001430 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001431 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001432 return number_of_reconfigurations_;
1433 }
1434
asaperssonfab67072017-04-04 05:51:49 -07001435 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001436 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001437 return min_transmit_bitrate_bps_;
1438 }
1439
Erik Språngd7329ca2019-02-21 21:19:53 +01001440 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001441 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001442 num_expected_layers_ = num_layers;
1443 }
1444
Erik Språngb7cb7b52019-02-26 15:52:33 +01001445 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001446 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001447 return last_capture_time_ms_;
1448 }
1449
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001450 const EncodedImage& GetLastEncodedImage() {
1451 MutexLock lock(&mutex_);
1452 return last_encoded_image_;
1453 }
1454
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001455 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001456 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001457 return std::move(last_encoded_image_data_);
1458 }
1459
Per Kjellanderdcef6412020-10-07 15:09:05 +02001460 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1461 MutexLock lock(&mutex_);
1462 return last_bitrate_allocation_;
1463 }
1464
1465 int number_of_bitrate_allocations() const {
1466 MutexLock lock(&mutex_);
1467 return number_of_bitrate_allocations_;
1468 }
1469
Per Kjellandera9434842020-10-15 17:53:22 +02001470 VideoLayersAllocation GetLastVideoLayersAllocation() {
1471 MutexLock lock(&mutex_);
1472 return last_layers_allocation_;
1473 }
1474
1475 int number_of_layers_allocations() const {
1476 MutexLock lock(&mutex_);
1477 return number_of_layers_allocations_;
1478 }
1479
perkj26091b12016-09-01 01:17:40 -07001480 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001481 Result OnEncodedImage(
1482 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001483 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001484 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001485 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001486 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001487 last_encoded_image_data_ = std::vector<uint8_t>(
1488 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001489 uint32_t timestamp = encoded_image.Timestamp();
1490 if (last_timestamp_ != timestamp) {
1491 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001492 last_width_ = encoded_image._encodedWidth;
1493 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001494 } else {
1495 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001496 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1497 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001498 }
1499 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001500 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001501 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001502 if (num_received_layers_ == num_expected_layers_) {
1503 encoded_frame_event_.Set();
1504 }
sprangb1ca0732017-02-01 08:38:12 -08001505 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001506 }
1507
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001508 void OnEncoderConfigurationChanged(
1509 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001510 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001511 VideoEncoderConfig::ContentType content_type,
1512 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001513 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001514 ++number_of_reconfigurations_;
1515 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1516 }
1517
Per Kjellanderdcef6412020-10-07 15:09:05 +02001518 void OnBitrateAllocationUpdated(
1519 const VideoBitrateAllocation& allocation) override {
1520 MutexLock lock(&mutex_);
1521 ++number_of_bitrate_allocations_;
1522 last_bitrate_allocation_ = allocation;
1523 }
1524
Per Kjellandera9434842020-10-15 17:53:22 +02001525 void OnVideoLayersAllocationUpdated(
1526 VideoLayersAllocation allocation) override {
1527 MutexLock lock(&mutex_);
1528 ++number_of_layers_allocations_;
1529 last_layers_allocation_ = allocation;
1530 rtc::StringBuilder log;
1531 for (const auto& layer : allocation.active_spatial_layers) {
1532 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1533 << "[";
1534 for (const auto target_bitrate :
1535 layer.target_bitrate_per_temporal_layer) {
1536 log << target_bitrate.kbps() << ",";
1537 }
1538 log << "]";
1539 }
Harald Alvestrand97597c02021-11-04 12:01:23 +00001540 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 17:53:22 +02001541 }
1542
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001543 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001544 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001545 TestEncoder* test_encoder_;
1546 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001547 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001548 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001549 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001550 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001551 uint32_t last_height_ = 0;
1552 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001553 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001554 size_t num_expected_layers_ = 1;
1555 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001556 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001557 int number_of_reconfigurations_ = 0;
1558 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001559 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1560 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001561 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1562 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001563 };
1564
Sergey Silkin5ee69672019-07-02 14:18:34 +02001565 class VideoBitrateAllocatorProxyFactory
1566 : public VideoBitrateAllocatorFactory {
1567 public:
1568 VideoBitrateAllocatorProxyFactory()
1569 : bitrate_allocator_factory_(
1570 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1571
1572 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1573 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001574 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001575 codec_config_ = codec;
1576 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1577 }
1578
1579 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001580 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001581 return codec_config_;
1582 }
1583
1584 private:
1585 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1586
Markus Handella3765182020-07-08 13:13:32 +02001587 mutable Mutex mutex_;
1588 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001589 };
1590
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001591 Clock* clock() { return time_controller_.GetClock(); }
1592 void AdvanceTime(TimeDelta duration) {
1593 time_controller_.AdvanceTime(duration);
1594 }
1595
1596 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1597
1598 protected:
1599 virtual TaskQueueFactory* GetTaskQueueFactory() {
1600 return time_controller_.GetTaskQueueFactory();
1601 }
1602
Jonas Orelandc7f691a2022-03-09 15:12:07 +01001603 test::ScopedKeyValueConfig field_trials_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001604 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001605 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001606 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001607 int codec_width_;
1608 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001609 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001610 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001611 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001612 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001613 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001614 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001615 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001616 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001617};
1618
mflodmancc3d4422017-08-03 08:27:51 -07001619TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001620 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001621 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001622 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001623 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001624 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001625 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001626 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001627}
1628
mflodmancc3d4422017-08-03 08:27:51 -07001629TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001630 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001631 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001632 // The encoder will cache up to one frame for a short duration. Adding two
1633 // frames means that the first frame will be dropped and the second frame will
1634 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001635 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001636 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001637 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Markus Handell28c71802021-11-08 10:11:55 +01001638 AdvanceTime(TimeDelta::Zero());
perkja49cbd32016-09-16 07:53:41 -07001639 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001640
Henrik Boström381d1092020-05-12 18:49:07 +02001641 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001642 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001643
Sebastian Janssona3177052018-04-10 13:05:49 +02001644 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001645 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001646 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1647
1648 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001649 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001650}
1651
mflodmancc3d4422017-08-03 08:27:51 -07001652TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001653 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001654 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001655 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001656 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001657
Henrik Boström381d1092020-05-12 18:49:07 +02001658 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001659 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1660
Sebastian Janssona3177052018-04-10 13:05:49 +02001661 // The encoder will cache up to one frame for a short duration. Adding two
1662 // frames means that the first frame will be dropped and the second frame will
1663 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001664 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001665 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001666
Henrik Boström381d1092020-05-12 18:49:07 +02001667 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001668 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001669 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001670 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1671 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001672 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001673}
1674
mflodmancc3d4422017-08-03 08:27:51 -07001675TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001676 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001677 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001678 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001679 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001680
1681 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001682 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001683
perkja49cbd32016-09-16 07:53:41 -07001684 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001685 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001686 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001687}
1688
mflodmancc3d4422017-08-03 08:27:51 -07001689TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001690 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001691 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001692
perkja49cbd32016-09-16 07:53:41 -07001693 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001694 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001695
mflodmancc3d4422017-08-03 08:27:51 -07001696 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001697 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001698 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001699 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1700 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001701}
1702
Markus Handell9a478b52021-11-18 16:07:01 +01001703TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
1704 test::FrameForwarder source;
1705 video_stream_encoder_->SetSource(&source,
1706 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02001707 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001708 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001709
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001710 int dropped_count = 0;
1711 stats_proxy_->SetDroppedFrameCallback(
1712 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1713 ++dropped_count;
1714 });
1715
Markus Handell9a478b52021-11-18 16:07:01 +01001716 source.IncomingCapturedFrame(CreateFrame(1, nullptr));
1717 source.IncomingCapturedFrame(CreateFrame(2, nullptr));
1718 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001719 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001720 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001721}
1722
Henrik Boström56db9ff2021-03-24 09:06:45 +01001723TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001724 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001725 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001726
1727 rtc::Event frame_destroyed_event;
1728 video_source_.IncomingCapturedFrame(
1729 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001730 WaitForEncodedFrame(1);
1731 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1732 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001733 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1734 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001735 video_stream_encoder_->Stop();
1736}
1737
Henrik Boström56db9ff2021-03-24 09:06:45 +01001738TEST_F(VideoStreamEncoderTest,
1739 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001740 // Use the cropping factory.
1741 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001742 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001743 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1744 kMaxPayloadLength);
1745 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1746
1747 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001748 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001749 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001750 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1751 WaitForEncodedFrame(1);
1752 // The encoder will have been configured once.
1753 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001754 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1755 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 06:59:12 -07001756
1757 // Now send in a fake frame that needs to be cropped as the width/height
1758 // aren't divisible by 4 (see CreateEncoderStreams above).
1759 rtc::Event frame_destroyed_event;
1760 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1761 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001762 WaitForEncodedFrame(2);
1763 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1764 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001765 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1766 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001767 video_stream_encoder_->Stop();
1768}
1769
Evan Shrubsole895556e2020-10-05 09:15:13 +02001770TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1771 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001772 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001773
1774 video_source_.IncomingCapturedFrame(
1775 CreateNV12Frame(1, codec_width_, codec_height_));
1776 WaitForEncodedFrame(1);
1777 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1778 fake_encoder_.GetLastInputPixelFormat());
1779 video_stream_encoder_->Stop();
1780}
1781
Henrik Boström56db9ff2021-03-24 09:06:45 +01001782TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001783 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001784 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001785
1786 fake_encoder_.SetPreferredPixelFormats({});
1787
1788 rtc::Event frame_destroyed_event;
1789 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1790 1, &frame_destroyed_event, codec_width_, codec_height_));
1791 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001792 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001793 fake_encoder_.GetLastInputPixelFormat());
1794 video_stream_encoder_->Stop();
1795}
1796
Henrik Boström56db9ff2021-03-24 09:06:45 +01001797TEST_F(VideoStreamEncoderTest,
1798 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001799 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001800 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001801
1802 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1803
1804 rtc::Event frame_destroyed_event;
1805 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1806 1, &frame_destroyed_event, codec_width_, codec_height_));
1807 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001808 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001809 fake_encoder_.GetLastInputPixelFormat());
1810 video_stream_encoder_->Stop();
1811}
1812
Henrik Boström56db9ff2021-03-24 09:06:45 +01001813TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001814 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001815 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001816
1817 // Fake NV12 native frame does not allow mapping to I444.
1818 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1819
1820 rtc::Event frame_destroyed_event;
1821 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1822 1, &frame_destroyed_event, codec_width_, codec_height_));
1823 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001824 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001825 fake_encoder_.GetLastInputPixelFormat());
1826 video_stream_encoder_->Stop();
1827}
1828
Henrik Boström56db9ff2021-03-24 09:06:45 +01001829TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001830 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001831 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001832
1833 rtc::Event frame_destroyed_event;
1834 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1835 1, &frame_destroyed_event, codec_width_, codec_height_));
1836 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001837 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001838 fake_encoder_.GetLastInputPixelFormat());
1839 video_stream_encoder_->Stop();
1840}
1841
Ying Wang9b881ab2020-02-07 14:29:32 +01001842TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001843 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001844 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001845 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1846 WaitForEncodedFrame(1);
1847
Henrik Boström381d1092020-05-12 18:49:07 +02001848 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001849 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001850 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1851 // frames. Adding two frames means that the first frame will be dropped and
1852 // the second frame will be sent to the encoder.
1853 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1854 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1855 WaitForEncodedFrame(3);
1856 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1857 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1858 WaitForEncodedFrame(5);
1859 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1860 video_stream_encoder_->Stop();
1861}
1862
mflodmancc3d4422017-08-03 08:27:51 -07001863TEST_F(VideoStreamEncoderTest,
1864 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001865 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001866 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001867 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001868
1869 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001870 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001871 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001872 // The encoder will have been configured once when the first frame is
1873 // received.
1874 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001875
1876 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001877 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001878 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001879 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001880 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001881
1882 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001883 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001884 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001885 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001886 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001887
mflodmancc3d4422017-08-03 08:27:51 -07001888 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001889}
1890
mflodmancc3d4422017-08-03 08:27:51 -07001891TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001892 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001893 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001894
1895 // Capture a frame and wait for it to synchronize with the encoder thread.
1896 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001897 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001898 // The encoder will have been configured once.
1899 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001900 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1901 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-02 23:45:26 -07001902
1903 codec_width_ *= 2;
1904 codec_height_ *= 2;
1905 // Capture a frame with a higher resolution and wait for it to synchronize
1906 // with the encoder thread.
1907 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001908 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 10:07:11 +02001909 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1910 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 21:37:57 +01001911 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001912
mflodmancc3d4422017-08-03 08:27:51 -07001913 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001914}
1915
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001916TEST_F(VideoStreamEncoderTest,
1917 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001918 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001919 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001920
1921 // Capture a frame and wait for it to synchronize with the encoder thread.
1922 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1923 WaitForEncodedFrame(1);
1924
1925 VideoEncoderConfig video_encoder_config;
1926 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1927 // Changing the max payload data length recreates encoder.
1928 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1929 kMaxPayloadLength / 2);
1930
1931 // Capture a frame and wait for it to synchronize with the encoder thread.
1932 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1933 WaitForEncodedFrame(2);
1934 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1935
1936 video_stream_encoder_->Stop();
1937}
1938
Sergey Silkin5ee69672019-07-02 14:18:34 +02001939TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001940 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001941 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001942
1943 VideoEncoderConfig video_encoder_config;
1944 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02001945 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1946 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001947 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1948 kMaxPayloadLength);
1949
1950 // Capture a frame and wait for it to synchronize with the encoder thread.
1951 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1952 WaitForEncodedFrame(1);
1953 // The encoder will have been configured once when the first frame is
1954 // received.
1955 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001956 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001957 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001958 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001959 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1960
Sergey Silkin6456e352019-07-08 17:56:40 +02001961 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1962 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 10:07:11 +02001963 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1964 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001965 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1966 kMaxPayloadLength);
1967
1968 // Capture a frame and wait for it to synchronize with the encoder thread.
1969 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1970 WaitForEncodedFrame(2);
1971 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1972 // Bitrate limits have changed - rate allocator should be reconfigured,
1973 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 10:07:11 +02001974 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001975 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001976 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001977 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001978 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001979
1980 video_stream_encoder_->Stop();
1981}
1982
Sergey Silkin6456e352019-07-08 17:56:40 +02001983TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001984 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001985 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001986 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001987
Sergey Silkincd02eba2020-01-20 14:48:40 +01001988 const uint32_t kMinEncBitrateKbps = 100;
1989 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001990 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001991 /*frame_size_pixels=*/codec_width_ * codec_height_,
1992 /*min_start_bitrate_bps=*/0,
1993 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1994 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001995 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1996
Sergey Silkincd02eba2020-01-20 14:48:40 +01001997 VideoEncoderConfig video_encoder_config;
1998 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1999 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
2000 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2001 (kMinEncBitrateKbps + 1) * 1000;
2002 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2003 kMaxPayloadLength);
2004
2005 // When both encoder and app provide bitrate limits, the intersection of
2006 // provided sets should be used.
2007 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2008 WaitForEncodedFrame(1);
2009 EXPECT_EQ(kMaxEncBitrateKbps,
2010 bitrate_allocator_factory_.codec_config().maxBitrate);
2011 EXPECT_EQ(kMinEncBitrateKbps + 1,
2012 bitrate_allocator_factory_.codec_config().minBitrate);
2013
2014 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
2015 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2016 (kMinEncBitrateKbps - 1) * 1000;
2017 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2018 kMaxPayloadLength);
2019 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002020 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002021 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002022 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002023 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002024 bitrate_allocator_factory_.codec_config().minBitrate);
2025
Sergey Silkincd02eba2020-01-20 14:48:40 +01002026 video_stream_encoder_->Stop();
2027}
2028
2029TEST_F(VideoStreamEncoderTest,
2030 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02002031 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002032 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002033
2034 const uint32_t kMinAppBitrateKbps = 100;
2035 const uint32_t kMaxAppBitrateKbps = 200;
2036 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
2037 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
2038 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2039 /*frame_size_pixels=*/codec_width_ * codec_height_,
2040 /*min_start_bitrate_bps=*/0,
2041 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
2042 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
2043 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2044
2045 VideoEncoderConfig video_encoder_config;
2046 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2047 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
2048 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2049 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002050 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2051 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002052
Sergey Silkincd02eba2020-01-20 14:48:40 +01002053 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2054 WaitForEncodedFrame(1);
2055 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002056 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002057 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002058 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02002059
2060 video_stream_encoder_->Stop();
2061}
2062
2063TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002064 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02002065 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002066 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02002067
2068 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002069 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002070 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002071 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002072 fake_encoder_.SetResolutionBitrateLimits(
2073 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
2074
2075 VideoEncoderConfig video_encoder_config;
2076 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2077 video_encoder_config.max_bitrate_bps = 0;
2078 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2079 kMaxPayloadLength);
2080
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002081 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002082 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2083 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002084 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2085 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002086 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2087 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2088
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002089 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002090 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2091 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002092 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2093 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002094 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2095 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2096
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002097 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02002098 // encoder for 360p should be used.
2099 video_source_.IncomingCapturedFrame(
2100 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2101 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002102 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2103 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002104 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2105 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2106
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002107 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02002108 // ignored.
2109 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2110 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002111 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2112 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002113 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2114 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002115 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2116 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002117 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2118 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2119
2120 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2121 // for 270p should be used.
2122 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2123 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002124 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2125 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002126 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2127 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2128
2129 video_stream_encoder_->Stop();
2130}
2131
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002132TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002133 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002134 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002135
2136 VideoEncoderConfig video_encoder_config;
2137 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2138 video_encoder_config.max_bitrate_bps = 0;
2139 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2140 kMaxPayloadLength);
2141
2142 // Encode 720p frame to get the default encoder target bitrate.
2143 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2144 WaitForEncodedFrame(1);
2145 const uint32_t kDefaultTargetBitrateFor720pKbps =
2146 bitrate_allocator_factory_.codec_config()
2147 .simulcastStream[0]
2148 .targetBitrate;
2149
2150 // Set the max recommended encoder bitrate to something lower than the default
2151 // target bitrate.
2152 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2153 1280 * 720, 10 * 1000, 10 * 1000,
2154 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2155 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2156
2157 // Change resolution to trigger encoder reinitialization.
2158 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2159 WaitForEncodedFrame(2);
2160 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2161 WaitForEncodedFrame(3);
2162
2163 // Ensure the target bitrate is capped by the max bitrate.
2164 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2165 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2166 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2167 .simulcastStream[0]
2168 .targetBitrate *
2169 1000,
2170 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2171
2172 video_stream_encoder_->Stop();
2173}
2174
Åsa Perssona7e34d32021-01-20 15:36:13 +01002175TEST_F(VideoStreamEncoderTest,
2176 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2177 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2178 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2179 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2180 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2181 fake_encoder_.SetResolutionBitrateLimits(
2182 {kEncoderLimits270p, kEncoderLimits360p});
2183
2184 // Two streams, highest stream active.
2185 VideoEncoderConfig config;
2186 const int kNumStreams = 2;
2187 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2188 config.max_bitrate_bps = 0;
2189 config.simulcast_layers[0].active = false;
2190 config.simulcast_layers[1].active = true;
2191 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002192 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002193 "VP8", /*max qp*/ 56, /*screencast*/ false,
2194 /*screenshare enabled*/ false);
2195 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2196
2197 // The encoder bitrate limits for 270p should be used.
2198 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2199 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002200 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002201 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002202 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002203 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002204 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002205
2206 // The encoder bitrate limits for 360p should be used.
2207 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2208 EXPECT_FALSE(WaitForFrame(1000));
2209 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002210 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002211 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002212 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002213
2214 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2215 video_source_.IncomingCapturedFrame(
2216 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2217 EXPECT_FALSE(WaitForFrame(1000));
2218 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002219 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002220 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002221 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002222
2223 // Resolution higher than 360p. Encoder limits should be ignored.
2224 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2225 EXPECT_FALSE(WaitForFrame(1000));
2226 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002227 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002228 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002229 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002230 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002231 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002232 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002233 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002234
2235 // Resolution lower than 270p. The encoder limits for 270p should be used.
2236 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2237 EXPECT_FALSE(WaitForFrame(1000));
2238 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002239 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002240 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002241 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002242
2243 video_stream_encoder_->Stop();
2244}
2245
2246TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002247 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2248 // Two streams, highest stream active.
2249 VideoEncoderConfig config;
2250 const int kNumStreams = 2;
2251 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2252 config.max_bitrate_bps = 0;
2253 config.simulcast_layers[0].active = false;
2254 config.simulcast_layers[1].active = true;
2255 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002256 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002257 "VP8", /*max qp*/ 56, /*screencast*/ false,
2258 /*screenshare enabled*/ false);
2259 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2260
2261 // Default bitrate limits for 270p should be used.
2262 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2263 kDefaultLimits270p =
2264 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002265 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002266 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2267 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002268 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 10:39:51 +01002269 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002270 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002271 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002272 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002273
2274 // Default bitrate limits for 360p should be used.
2275 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2276 kDefaultLimits360p =
2277 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002278 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002279 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2280 EXPECT_FALSE(WaitForFrame(1000));
2281 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002282 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002283 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002284 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002285
2286 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2287 video_source_.IncomingCapturedFrame(
2288 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2289 EXPECT_FALSE(WaitForFrame(1000));
2290 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002291 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002292 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002293 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002294
2295 // Default bitrate limits for 540p should be used.
2296 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2297 kDefaultLimits540p =
2298 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002299 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002300 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2301 EXPECT_FALSE(WaitForFrame(1000));
2302 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002303 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002304 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002305 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002306
2307 video_stream_encoder_->Stop();
2308}
2309
2310TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002311 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2312 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2313 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2314 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2315 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2316 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2317 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2318 fake_encoder_.SetResolutionBitrateLimits(
2319 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2320
2321 // Three streams, middle stream active.
2322 VideoEncoderConfig config;
2323 const int kNumStreams = 3;
2324 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2325 config.simulcast_layers[0].active = false;
2326 config.simulcast_layers[1].active = true;
2327 config.simulcast_layers[2].active = false;
2328 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002329 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002330 "VP8", /*max qp*/ 56, /*screencast*/ false,
2331 /*screenshare enabled*/ false);
2332 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2333
2334 // The encoder bitrate limits for 360p should be used.
2335 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2336 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002337 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002338 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002339 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002340 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002341 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002342
2343 // The encoder bitrate limits for 270p should be used.
2344 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2345 EXPECT_FALSE(WaitForFrame(1000));
2346 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002347 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002348 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002349 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002350
2351 video_stream_encoder_->Stop();
2352}
2353
2354TEST_F(VideoStreamEncoderTest,
2355 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2356 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2357 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2358 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2359 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2360 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2361 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2362 fake_encoder_.SetResolutionBitrateLimits(
2363 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2364
2365 // Three streams, lowest stream active.
2366 VideoEncoderConfig config;
2367 const int kNumStreams = 3;
2368 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2369 config.simulcast_layers[0].active = true;
2370 config.simulcast_layers[1].active = false;
2371 config.simulcast_layers[2].active = false;
2372 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002373 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002374 "VP8", /*max qp*/ 56, /*screencast*/ false,
2375 /*screenshare enabled*/ false);
2376 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2377
2378 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2379 // on lowest stream, limits for 270p should not be used
2380 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2381 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002382 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002383 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002384 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002385 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002386 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002387
2388 video_stream_encoder_->Stop();
2389}
2390
2391TEST_F(VideoStreamEncoderTest,
2392 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2393 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2394 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2395 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2396 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2397 fake_encoder_.SetResolutionBitrateLimits(
2398 {kEncoderLimits270p, kEncoderLimits360p});
2399 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2400
2401 // Two streams, highest stream active.
2402 VideoEncoderConfig config;
2403 const int kNumStreams = 2;
2404 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2405 config.simulcast_layers[0].active = false;
2406 config.simulcast_layers[1].active = true;
2407 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2408 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002409 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002410 "VP8", /*max qp*/ 56, /*screencast*/ false,
2411 /*screenshare enabled*/ false);
2412 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2413
2414 // The encoder bitrate limits for 270p should be used.
2415 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2416 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002417 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002418 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002419 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002420 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002421 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002422
2423 // The max configured bitrate is less than the encoder limit for 360p.
2424 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2425 EXPECT_FALSE(WaitForFrame(1000));
2426 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002427 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002428 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002429 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002430
2431 video_stream_encoder_->Stop();
2432}
2433
mflodmancc3d4422017-08-03 08:27:51 -07002434TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002435 EXPECT_TRUE(video_source_.has_sinks());
2436 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002437 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002438 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002439 EXPECT_FALSE(video_source_.has_sinks());
2440 EXPECT_TRUE(new_video_source.has_sinks());
2441
mflodmancc3d4422017-08-03 08:27:51 -07002442 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002443}
2444
mflodmancc3d4422017-08-03 08:27:51 -07002445TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002446 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002447 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002448 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002449 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002450}
2451
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002452class ResolutionAlignmentTest
2453 : public VideoStreamEncoderTest,
2454 public ::testing::WithParamInterface<
2455 ::testing::tuple<int, std::vector<double>>> {
2456 public:
2457 ResolutionAlignmentTest()
2458 : requested_alignment_(::testing::get<0>(GetParam())),
2459 scale_factors_(::testing::get<1>(GetParam())) {}
2460
2461 protected:
2462 const int requested_alignment_;
2463 const std::vector<double> scale_factors_;
2464};
2465
2466INSTANTIATE_TEST_SUITE_P(
2467 AlignmentAndScaleFactors,
2468 ResolutionAlignmentTest,
2469 ::testing::Combine(
2470 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2471 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2472 std::vector<double>{-1.0, -1.0},
2473 std::vector<double>{-1.0, -1.0, -1.0},
2474 std::vector<double>{4.0, 2.0, 1.0},
2475 std::vector<double>{9999.0, -1.0, 1.0},
2476 std::vector<double>{3.99, 2.01, 1.0},
2477 std::vector<double>{4.9, 1.7, 1.25},
2478 std::vector<double>{10.0, 4.0, 3.0},
2479 std::vector<double>{1.75, 3.5},
2480 std::vector<double>{1.5, 2.5},
2481 std::vector<double>{1.3, 1.0})));
2482
2483TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2484 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002485 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002486 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2487 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2488
2489 // Fill config with the scaling factor by which to reduce encoding size.
2490 const int num_streams = scale_factors_.size();
2491 VideoEncoderConfig config;
2492 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2493 for (int i = 0; i < num_streams; ++i) {
2494 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2495 }
2496 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002497 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002498 "VP8", /*max qp*/ 56, /*screencast*/ false,
2499 /*screenshare enabled*/ false);
2500 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2501
Henrik Boström381d1092020-05-12 18:49:07 +02002502 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002503 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2504 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002505 // Wait for all layers before triggering event.
2506 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002507
2508 // On the 1st frame, we should have initialized the encoder and
2509 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002510 int64_t timestamp_ms = kFrameIntervalMs;
2511 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2512 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002513 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002514
2515 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2516 // (It's up the to the encoder to potentially drop the previous frame,
2517 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002518 timestamp_ms += kFrameIntervalMs;
2519 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2520 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002521 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002522
Asa Persson606d3cb2021-10-04 10:07:11 +02002523 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002524 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2525 // Frame size should be a multiple of the requested alignment.
2526 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2527 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2528 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2529 // Aspect ratio should match.
2530 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2531 codec.height * codec.simulcastStream[i].width);
2532 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002533
2534 video_stream_encoder_->Stop();
2535}
2536
Jonathan Yubc771b72017-12-08 17:04:29 -08002537TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2538 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002539 const int kWidth = 1280;
2540 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002541
2542 // We rely on the automatic resolution adaptation, but we handle framerate
2543 // adaptation manually by mocking the stats proxy.
2544 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002545
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002546 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002547 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002548 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002549 video_stream_encoder_->SetSource(&video_source_,
2550 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002551 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002552 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002553 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002554 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2555
Jonathan Yubc771b72017-12-08 17:04:29 -08002556 // Adapt down as far as possible.
2557 rtc::VideoSinkWants last_wants;
2558 int64_t t = 1;
2559 int loop_count = 0;
2560 do {
2561 ++loop_count;
2562 last_wants = video_source_.sink_wants();
2563
2564 // Simulate the framerate we've been asked to adapt to.
2565 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2566 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2567 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2568 mock_stats.input_frame_rate = fps;
2569 stats_proxy_->SetMockStats(mock_stats);
2570
2571 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2572 sink_.WaitForEncodedFrame(t);
2573 t += frame_interval_ms;
2574
mflodmancc3d4422017-08-03 08:27:51 -07002575 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002576 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002577 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002578 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2579 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002580 } while (video_source_.sink_wants().max_pixel_count <
2581 last_wants.max_pixel_count ||
2582 video_source_.sink_wants().max_framerate_fps <
2583 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002584
Jonathan Yubc771b72017-12-08 17:04:29 -08002585 // Verify that we've adapted all the way down.
2586 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002587 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002588 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2589 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002590 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002591 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2592 *video_source_.last_sent_height());
2593 EXPECT_EQ(kMinBalancedFramerateFps,
2594 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002595
Jonathan Yubc771b72017-12-08 17:04:29 -08002596 // Adapt back up the same number of times we adapted down.
2597 for (int i = 0; i < loop_count - 1; ++i) {
2598 last_wants = video_source_.sink_wants();
2599
2600 // Simulate the framerate we've been asked to adapt to.
2601 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2602 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2603 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2604 mock_stats.input_frame_rate = fps;
2605 stats_proxy_->SetMockStats(mock_stats);
2606
2607 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2608 sink_.WaitForEncodedFrame(t);
2609 t += frame_interval_ms;
2610
Henrik Boström91aa7322020-04-28 12:24:33 +02002611 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002612 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002613 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002614 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2615 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002616 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2617 last_wants.max_pixel_count ||
2618 video_source_.sink_wants().max_framerate_fps >
2619 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002620 }
2621
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002622 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002623 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002624 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002625 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2626 EXPECT_EQ((loop_count - 1) * 2,
2627 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002628
mflodmancc3d4422017-08-03 08:27:51 -07002629 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002630}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002631
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002632TEST_F(VideoStreamEncoderTest,
2633 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02002634 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2635 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002636 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002637
2638 const int kFrameWidth = 1280;
2639 const int kFrameHeight = 720;
2640
2641 int64_t ntp_time = kFrameIntervalMs;
2642
2643 // Force an input frame rate to be available, or the adaptation call won't
2644 // know what framerate to adapt form.
2645 const int kInputFps = 30;
2646 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2647 stats.input_frame_rate = kInputFps;
2648 stats_proxy_->SetMockStats(stats);
2649
2650 video_source_.set_adaptation_enabled(true);
2651 video_stream_encoder_->SetSource(
2652 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002653 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002654 video_source_.IncomingCapturedFrame(
2655 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2656 sink_.WaitForEncodedFrame(ntp_time);
2657 ntp_time += kFrameIntervalMs;
2658
2659 // Trigger CPU overuse.
2660 video_stream_encoder_->TriggerCpuOveruse();
2661 video_source_.IncomingCapturedFrame(
2662 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2663 sink_.WaitForEncodedFrame(ntp_time);
2664 ntp_time += kFrameIntervalMs;
2665
2666 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2667 EXPECT_EQ(std::numeric_limits<int>::max(),
2668 video_source_.sink_wants().max_pixel_count);
2669 // Some framerate constraint should be set.
2670 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2671 EXPECT_LT(restricted_fps, kInputFps);
2672 video_source_.IncomingCapturedFrame(
2673 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2674 sink_.WaitForEncodedFrame(ntp_time);
2675 ntp_time += 100;
2676
Henrik Boström2671dac2020-05-19 16:29:09 +02002677 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002678 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2679 // Give the encoder queue time to process the change in degradation preference
2680 // by waiting for an encoded frame.
2681 video_source_.IncomingCapturedFrame(
2682 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2683 sink_.WaitForEncodedFrame(ntp_time);
2684 ntp_time += kFrameIntervalMs;
2685
2686 video_stream_encoder_->TriggerQualityLow();
2687 video_source_.IncomingCapturedFrame(
2688 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2689 sink_.WaitForEncodedFrame(ntp_time);
2690 ntp_time += kFrameIntervalMs;
2691
2692 // Some resolution constraint should be set.
2693 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2694 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2695 kFrameWidth * kFrameHeight);
2696 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2697
2698 int pixel_count = video_source_.sink_wants().max_pixel_count;
2699 // Triggering a CPU underuse should not change the sink wants since it has
2700 // not been overused for resolution since we changed degradation preference.
2701 video_stream_encoder_->TriggerCpuUnderuse();
2702 video_source_.IncomingCapturedFrame(
2703 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2704 sink_.WaitForEncodedFrame(ntp_time);
2705 ntp_time += kFrameIntervalMs;
2706 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2707 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2708
Evan Shrubsole64469032020-06-11 10:45:29 +02002709 // Change the degradation preference back. CPU underuse should not adapt since
2710 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002711 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002712 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2713 video_source_.IncomingCapturedFrame(
2714 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2715 sink_.WaitForEncodedFrame(ntp_time);
2716 ntp_time += 100;
2717 // Resolution adaptations is gone after changing degradation preference.
2718 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2719 EXPECT_EQ(std::numeric_limits<int>::max(),
2720 video_source_.sink_wants().max_pixel_count);
2721 // The fps adaptation from above is now back.
2722 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2723
2724 // Trigger CPU underuse.
2725 video_stream_encoder_->TriggerCpuUnderuse();
2726 video_source_.IncomingCapturedFrame(
2727 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2728 sink_.WaitForEncodedFrame(ntp_time);
2729 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002730 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2731
2732 // Trigger QP underuse, fps should return to normal.
2733 video_stream_encoder_->TriggerQualityHigh();
2734 video_source_.IncomingCapturedFrame(
2735 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2736 sink_.WaitForEncodedFrame(ntp_time);
2737 ntp_time += kFrameIntervalMs;
2738 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002739
2740 video_stream_encoder_->Stop();
2741}
2742
mflodmancc3d4422017-08-03 08:27:51 -07002743TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002744 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002745 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002746 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002747
sprangc5d62e22017-04-02 23:53:04 -07002748 const int kFrameWidth = 1280;
2749 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002750
Åsa Persson8c1bf952018-09-13 10:42:19 +02002751 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002752
kthelgason5e13d412016-12-01 03:59:51 -08002753 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002754 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002755 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002756 frame_timestamp += kFrameIntervalMs;
2757
perkj803d97f2016-11-01 11:45:46 -07002758 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002759 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002760 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002761 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002762 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002763 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002764
asapersson0944a802017-04-07 00:57:58 -07002765 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002766 // wanted resolution.
2767 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2768 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2769 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002770 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002771
2772 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002773 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002774 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002775 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002776 // Give the encoder queue time to process the change in degradation preference
2777 // by waiting for an encoded frame.
2778 new_video_source.IncomingCapturedFrame(
2779 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2780 sink_.WaitForEncodedFrame(frame_timestamp);
2781 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002782 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002783 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002784
sprangc5d62e22017-04-02 23:53:04 -07002785 // Force an input frame rate to be available, or the adaptation call won't
2786 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002787 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002788 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002789 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002790 stats_proxy_->SetMockStats(stats);
2791
mflodmancc3d4422017-08-03 08:27:51 -07002792 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002793 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002794 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002795 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002796 frame_timestamp += kFrameIntervalMs;
2797
2798 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002799 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002800 EXPECT_EQ(std::numeric_limits<int>::max(),
2801 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002802 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002803
asapersson02465b82017-04-10 01:12:52 -07002804 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002805 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2806 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002807 // Give the encoder queue time to process the change in degradation preference
2808 // by waiting for an encoded frame.
2809 new_video_source.IncomingCapturedFrame(
2810 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2811 sink_.WaitForEncodedFrame(frame_timestamp);
2812 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002813 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002814
mflodmancc3d4422017-08-03 08:27:51 -07002815 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002816 new_video_source.IncomingCapturedFrame(
2817 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002818 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002819 frame_timestamp += kFrameIntervalMs;
2820
2821 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002822 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002823
2824 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002825 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002826 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002827 // Give the encoder queue time to process the change in degradation preference
2828 // by waiting for an encoded frame.
2829 new_video_source.IncomingCapturedFrame(
2830 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2831 sink_.WaitForEncodedFrame(frame_timestamp);
2832 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002833 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2834 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002835 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002836 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002837
2838 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002839 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002840 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002841 // Give the encoder queue time to process the change in degradation preference
2842 // by waiting for an encoded frame.
2843 new_video_source.IncomingCapturedFrame(
2844 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2845 sink_.WaitForEncodedFrame(frame_timestamp);
2846 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002847 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2848 EXPECT_EQ(std::numeric_limits<int>::max(),
2849 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002850 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002851
mflodmancc3d4422017-08-03 08:27:51 -07002852 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002853}
2854
mflodmancc3d4422017-08-03 08:27:51 -07002855TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002856 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002857 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002858
asaperssonfab67072017-04-04 05:51:49 -07002859 const int kWidth = 1280;
2860 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002861 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002862 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002863 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2864 EXPECT_FALSE(stats.bw_limited_resolution);
2865 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2866
2867 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002868 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002869 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002870 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002871
2872 stats = stats_proxy_->GetStats();
2873 EXPECT_TRUE(stats.bw_limited_resolution);
2874 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2875
2876 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002877 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002878 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002879 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002880
2881 stats = stats_proxy_->GetStats();
2882 EXPECT_FALSE(stats.bw_limited_resolution);
2883 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2884 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2885
mflodmancc3d4422017-08-03 08:27:51 -07002886 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002887}
2888
mflodmancc3d4422017-08-03 08:27:51 -07002889TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002890 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002891 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002892
2893 const int kWidth = 1280;
2894 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002895 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002896 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002897 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2898 EXPECT_FALSE(stats.cpu_limited_resolution);
2899 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2900
2901 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002902 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002903 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002904 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002905
2906 stats = stats_proxy_->GetStats();
2907 EXPECT_TRUE(stats.cpu_limited_resolution);
2908 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2909
2910 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002911 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002912 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002913 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002914
2915 stats = stats_proxy_->GetStats();
2916 EXPECT_FALSE(stats.cpu_limited_resolution);
2917 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002918 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002919
mflodmancc3d4422017-08-03 08:27:51 -07002920 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002921}
2922
mflodmancc3d4422017-08-03 08:27:51 -07002923TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002924 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002925 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002926
asaperssonfab67072017-04-04 05:51:49 -07002927 const int kWidth = 1280;
2928 const int kHeight = 720;
2929 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002930 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002931 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002932 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002933 EXPECT_FALSE(stats.cpu_limited_resolution);
2934 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2935
asaperssonfab67072017-04-04 05:51:49 -07002936 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002937 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002938 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002939 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002940 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002941 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002942 EXPECT_TRUE(stats.cpu_limited_resolution);
2943 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2944
2945 // Set new source with adaptation still enabled.
2946 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002947 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002948 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002949
asaperssonfab67072017-04-04 05:51:49 -07002950 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002951 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002952 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002953 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002954 EXPECT_TRUE(stats.cpu_limited_resolution);
2955 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2956
2957 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002958 video_stream_encoder_->SetSource(&new_video_source,
2959 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002960
asaperssonfab67072017-04-04 05:51:49 -07002961 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002962 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002963 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002964 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002965 EXPECT_FALSE(stats.cpu_limited_resolution);
2966 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2967
2968 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002969 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002970 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002971
asaperssonfab67072017-04-04 05:51:49 -07002972 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002973 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002974 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002975 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002976 EXPECT_TRUE(stats.cpu_limited_resolution);
2977 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2978
asaperssonfab67072017-04-04 05:51:49 -07002979 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002980 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002981 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002982 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002983 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002984 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002985 EXPECT_FALSE(stats.cpu_limited_resolution);
2986 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002987 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002988
mflodmancc3d4422017-08-03 08:27:51 -07002989 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002990}
2991
mflodmancc3d4422017-08-03 08:27:51 -07002992TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002993 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002994 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002995
asaperssonfab67072017-04-04 05:51:49 -07002996 const int kWidth = 1280;
2997 const int kHeight = 720;
2998 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002999 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003000 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003001 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003002 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003003 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003004
3005 // Set new source with adaptation still enabled.
3006 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003007 video_stream_encoder_->SetSource(&new_video_source,
3008 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08003009
asaperssonfab67072017-04-04 05:51:49 -07003010 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003011 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08003012 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003013 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003014 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003015 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003016
asaperssonfab67072017-04-04 05:51:49 -07003017 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003018 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003019 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003020 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003021 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003022 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003023 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003024 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003025
asaperssonfab67072017-04-04 05:51:49 -07003026 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003027 video_stream_encoder_->SetSource(&new_video_source,
3028 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08003029
asaperssonfab67072017-04-04 05:51:49 -07003030 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003031 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003032 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003033 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003034 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003035 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003036
asapersson02465b82017-04-10 01:12:52 -07003037 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07003038 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003039 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003040
asaperssonfab67072017-04-04 05:51:49 -07003041 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003042 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08003043 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003044 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003045 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003046 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
3047 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003048
mflodmancc3d4422017-08-03 08:27:51 -07003049 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003050}
3051
mflodmancc3d4422017-08-03 08:27:51 -07003052TEST_F(VideoStreamEncoderTest,
3053 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02003054 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003055 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07003056
3057 const int kWidth = 1280;
3058 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02003059 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07003060 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003061 video_source_.IncomingCapturedFrame(
3062 CreateFrame(timestamp_ms, kWidth, kHeight));
3063 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003064 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3065 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3066 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3067
3068 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003069 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003070 timestamp_ms += kFrameIntervalMs;
3071 video_source_.IncomingCapturedFrame(
3072 CreateFrame(timestamp_ms, kWidth, kHeight));
3073 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003074 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3075 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3076 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3077
3078 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07003079 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003080 timestamp_ms += kFrameIntervalMs;
3081 video_source_.IncomingCapturedFrame(
3082 CreateFrame(timestamp_ms, kWidth, kHeight));
3083 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003084 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3085 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3086 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3087
Niels Möller4db138e2018-04-19 09:04:13 +02003088 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07003089 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003090
3091 VideoEncoderConfig video_encoder_config;
3092 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3093 // Make format different, to force recreation of encoder.
3094 video_encoder_config.video_format.parameters["foo"] = "foo";
3095 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003096 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003097 timestamp_ms += kFrameIntervalMs;
3098 video_source_.IncomingCapturedFrame(
3099 CreateFrame(timestamp_ms, kWidth, kHeight));
3100 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003101 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3102 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3103 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3104
mflodmancc3d4422017-08-03 08:27:51 -07003105 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003106}
3107
mflodmancc3d4422017-08-03 08:27:51 -07003108TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003109 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003110 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003111 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003112
3113 const int kWidth = 1280;
3114 const int kHeight = 720;
3115 int sequence = 1;
3116
3117 // Enable BALANCED preference, no initial limitation.
3118 test::FrameForwarder source;
3119 video_stream_encoder_->SetSource(&source,
3120 webrtc::DegradationPreference::BALANCED);
3121 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3122 WaitForEncodedFrame(sequence++);
3123 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3124 EXPECT_FALSE(stats.cpu_limited_resolution);
3125 EXPECT_FALSE(stats.cpu_limited_framerate);
3126 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3127
3128 // Trigger CPU overuse, should now adapt down.
3129 video_stream_encoder_->TriggerCpuOveruse();
3130 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3131 WaitForEncodedFrame(sequence++);
3132 stats = stats_proxy_->GetStats();
3133 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3134
3135 // Set new degradation preference should clear restrictions since we changed
3136 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003137 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003138 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3139 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3140 WaitForEncodedFrame(sequence++);
3141 stats = stats_proxy_->GetStats();
3142 EXPECT_FALSE(stats.cpu_limited_resolution);
3143 EXPECT_FALSE(stats.cpu_limited_framerate);
3144 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3145
3146 // Force an input frame rate to be available, or the adaptation call won't
3147 // know what framerate to adapt from.
3148 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3149 mock_stats.input_frame_rate = 30;
3150 stats_proxy_->SetMockStats(mock_stats);
3151 video_stream_encoder_->TriggerCpuOveruse();
3152 stats_proxy_->ResetMockStats();
3153 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3154 WaitForEncodedFrame(sequence++);
3155
3156 // We have now adapted once.
3157 stats = stats_proxy_->GetStats();
3158 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3159
3160 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003161 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3162 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003163 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3164 WaitForEncodedFrame(sequence++);
3165 stats = stats_proxy_->GetStats();
3166 EXPECT_FALSE(stats.cpu_limited_resolution);
3167 EXPECT_FALSE(stats.cpu_limited_framerate);
3168 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3169
3170 video_stream_encoder_->Stop();
3171}
3172
3173TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003174 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003175 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003176 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003177
asapersson0944a802017-04-07 00:57:58 -07003178 const int kWidth = 1280;
3179 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003180 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003181
asaperssonfab67072017-04-04 05:51:49 -07003182 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003183 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003184 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003185 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003186 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003187 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3188
asapersson02465b82017-04-10 01:12:52 -07003189 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003190 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003191 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003192 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003193 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003194 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003195 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003196 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3197
3198 // Set new source with adaptation still enabled.
3199 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003200 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003201 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003202
3203 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003204 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003205 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003206 stats = stats_proxy_->GetStats();
3207 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003208 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003209 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3210
sprangc5d62e22017-04-02 23:53:04 -07003211 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003212 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003213 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003214 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003215 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003216 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003217 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003218 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003219 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003220 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003221 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3222
sprangc5d62e22017-04-02 23:53:04 -07003223 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003224 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003225 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3226 mock_stats.input_frame_rate = 30;
3227 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003228 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003229 stats_proxy_->ResetMockStats();
3230
3231 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003232 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003233 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003234
3235 // Framerate now adapted.
3236 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003237 EXPECT_FALSE(stats.cpu_limited_resolution);
3238 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003239 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3240
3241 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003242 video_stream_encoder_->SetSource(&new_video_source,
3243 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003244 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003245 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003246 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003247
3248 stats = stats_proxy_->GetStats();
3249 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003250 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003251 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3252
3253 // Try to trigger overuse. Should not succeed.
3254 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003255 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003256 stats_proxy_->ResetMockStats();
3257
3258 stats = stats_proxy_->GetStats();
3259 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003260 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003261 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3262
3263 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003264 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003265 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003266 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003267 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003268 stats = stats_proxy_->GetStats();
3269 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003270 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003271 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003272
3273 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003274 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003275 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003276 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003277 stats = stats_proxy_->GetStats();
3278 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003279 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003280 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3281
3282 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003283 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003284 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003285 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003286 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003287 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003288 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003289 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003290 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003291 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003292 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3293
3294 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003295 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003296 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003297 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003298 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003299 stats = stats_proxy_->GetStats();
3300 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003301 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003302 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003303 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003304
mflodmancc3d4422017-08-03 08:27:51 -07003305 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003306}
3307
mflodmancc3d4422017-08-03 08:27:51 -07003308TEST_F(VideoStreamEncoderTest,
3309 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003310 const int kWidth = 1280;
3311 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003312 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003313 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003314
asaperssonfab67072017-04-04 05:51:49 -07003315 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003316 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003317
asaperssonfab67072017-04-04 05:51:49 -07003318 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003319 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003320
asaperssonfab67072017-04-04 05:51:49 -07003321 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003322 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003323
asaperssonfab67072017-04-04 05:51:49 -07003324 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003325 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003326
kthelgason876222f2016-11-29 01:44:11 -08003327 // Expect a scale down.
3328 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003329 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003330
asapersson02465b82017-04-10 01:12:52 -07003331 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003332 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003333 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003334 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003335
asaperssonfab67072017-04-04 05:51:49 -07003336 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003337 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003338 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003339 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003340
asaperssonfab67072017-04-04 05:51:49 -07003341 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003342 EXPECT_EQ(std::numeric_limits<int>::max(),
3343 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003344
asaperssonfab67072017-04-04 05:51:49 -07003345 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003346 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003347 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003348 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003349
asapersson02465b82017-04-10 01:12:52 -07003350 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003351 EXPECT_EQ(std::numeric_limits<int>::max(),
3352 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003353
mflodmancc3d4422017-08-03 08:27:51 -07003354 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003355}
3356
mflodmancc3d4422017-08-03 08:27:51 -07003357TEST_F(VideoStreamEncoderTest,
3358 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003359 const int kWidth = 1280;
3360 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003361 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003362 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003363
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003364 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003365 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003366 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003367 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003368
3369 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003370 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003371 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003372 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3373 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3374
3375 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003376 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003377 EXPECT_THAT(source.sink_wants(),
3378 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003379 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3380 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3381 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3382
3383 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003384 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003385 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3386 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3387 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3388
mflodmancc3d4422017-08-03 08:27:51 -07003389 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003390}
3391
mflodmancc3d4422017-08-03 08:27:51 -07003392TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003393 const int kWidth = 1280;
3394 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003395 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003396 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003397
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003398 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003399 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003400 video_stream_encoder_->SetSource(&source,
3401 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003402 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3403 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003404 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003405
3406 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003407 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003408 EXPECT_THAT(source.sink_wants(),
3409 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003410 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3411 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3412 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3413
3414 // Trigger adapt down for same input resolution, expect no change.
3415 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3416 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003417 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003418 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3419 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3420 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3421
3422 // Trigger adapt down for larger input resolution, expect no change.
3423 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3424 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003425 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003426 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3427 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3428 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3429
mflodmancc3d4422017-08-03 08:27:51 -07003430 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003431}
3432
mflodmancc3d4422017-08-03 08:27:51 -07003433TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003434 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3435 const int kWidth = 640;
3436 const int kHeight = 360;
3437 const int64_t kFrameIntervalMs = 150;
3438 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003439 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003440
3441 // Enable BALANCED preference, no initial limitation.
3442 AdaptingFrameForwarder source(&time_controller_);
3443 source.set_adaptation_enabled(true);
3444 video_stream_encoder_->SetSource(&source,
3445 webrtc::DegradationPreference::BALANCED);
3446
3447 int64_t timestamp_ms = kFrameIntervalMs;
3448 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3449 sink_.WaitForEncodedFrame(kWidth, kHeight);
3450 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3451 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3452 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3453 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3454
3455 // Trigger adapt down, expect reduced fps (640x360@15fps).
3456 video_stream_encoder_->TriggerQualityLow();
3457 timestamp_ms += kFrameIntervalMs;
3458 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3459 sink_.WaitForEncodedFrame(timestamp_ms);
3460 EXPECT_THAT(source.sink_wants(),
3461 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3462 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3463 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3464 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3465
3466 // Source requests 270p, expect reduced resolution (480x270@15fps).
3467 source.OnOutputFormatRequest(480, 270);
3468 timestamp_ms += kFrameIntervalMs;
3469 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3470 WaitForEncodedFrame(480, 270);
3471 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3472
3473 // Trigger adapt down, expect reduced fps (480x270@10fps).
3474 video_stream_encoder_->TriggerQualityLow();
3475 timestamp_ms += kFrameIntervalMs;
3476 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3477 sink_.WaitForEncodedFrame(timestamp_ms);
3478 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3479 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3480 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3481 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3482
3483 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3484 source.OnOutputFormatRequest(320, 180);
3485 timestamp_ms += kFrameIntervalMs;
3486 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3487 WaitForEncodedFrame(320, 180);
3488 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3489
3490 // Trigger adapt down, expect reduced fps (320x180@7fps).
3491 video_stream_encoder_->TriggerQualityLow();
3492 timestamp_ms += kFrameIntervalMs;
3493 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3494 sink_.WaitForEncodedFrame(timestamp_ms);
3495 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3496 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3497 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3498 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3499
3500 // Source requests VGA, expect increased resolution (640x360@7fps).
3501 source.OnOutputFormatRequest(640, 360);
3502 timestamp_ms += kFrameIntervalMs;
3503 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3504 WaitForEncodedFrame(timestamp_ms);
3505 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3506
3507 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3508 video_stream_encoder_->TriggerQualityHigh();
3509 timestamp_ms += kFrameIntervalMs;
3510 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3511 WaitForEncodedFrame(timestamp_ms);
3512 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3513 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3514 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3515 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3516
3517 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3518 video_stream_encoder_->TriggerQualityHigh();
3519 timestamp_ms += kFrameIntervalMs;
3520 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3521 WaitForEncodedFrame(timestamp_ms);
3522 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3523 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3524 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3525 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3526
3527 // Trigger adapt up, expect increased fps (640x360@maxfps).
3528 video_stream_encoder_->TriggerQualityHigh();
3529 timestamp_ms += kFrameIntervalMs;
3530 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3531 WaitForEncodedFrame(timestamp_ms);
3532 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3533 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3534 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3535 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3536
3537 video_stream_encoder_->Stop();
3538}
3539
3540TEST_F(VideoStreamEncoderTest,
3541 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3542 const int kWidth = 1280;
3543 const int kHeight = 720;
3544 const int64_t kFrameIntervalMs = 150;
3545 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003546 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003547
3548 // Enable BALANCED preference, no initial limitation.
3549 AdaptingFrameForwarder source(&time_controller_);
3550 source.set_adaptation_enabled(true);
3551 video_stream_encoder_->SetSource(&source,
3552 webrtc::DegradationPreference::BALANCED);
3553
3554 int64_t timestamp_ms = kFrameIntervalMs;
3555 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3556 sink_.WaitForEncodedFrame(kWidth, kHeight);
3557 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3558 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3559 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3560 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3561
3562 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3563 video_stream_encoder_->TriggerQualityLow();
3564 timestamp_ms += kFrameIntervalMs;
3565 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3566 sink_.WaitForEncodedFrame(timestamp_ms);
3567 EXPECT_THAT(source.sink_wants(),
3568 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3569 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3570 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3571 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3572
3573 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3574 video_stream_encoder_->TriggerQualityLow();
3575 timestamp_ms += kFrameIntervalMs;
3576 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3577 sink_.WaitForEncodedFrame(timestamp_ms);
3578 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3579 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3580 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3581 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3582
3583 // Trigger adapt down, expect reduced fps (640x360@15fps).
3584 video_stream_encoder_->TriggerQualityLow();
3585 timestamp_ms += kFrameIntervalMs;
3586 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3587 WaitForEncodedFrame(timestamp_ms);
3588 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3589 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3590 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3591 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3592
3593 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3594 source.OnOutputFormatRequest(320, 180);
3595 timestamp_ms += kFrameIntervalMs;
3596 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3597 WaitForEncodedFrame(320, 180);
3598 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3599 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3600
3601 // Trigger adapt down, expect reduced fps (320x180@7fps).
3602 video_stream_encoder_->TriggerCpuOveruse();
3603 timestamp_ms += kFrameIntervalMs;
3604 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3605 WaitForEncodedFrame(timestamp_ms);
3606 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3607 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3608 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3609 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3610 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3611 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3612 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3613
3614 // Source requests HD, expect increased resolution (640x360@7fps).
3615 source.OnOutputFormatRequest(1280, 720);
3616 timestamp_ms += kFrameIntervalMs;
3617 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3618 WaitForEncodedFrame(timestamp_ms);
3619 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3620 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3621
3622 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3623 video_stream_encoder_->TriggerCpuUnderuse();
3624 timestamp_ms += kFrameIntervalMs;
3625 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3626 WaitForEncodedFrame(timestamp_ms);
3627 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3628 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3629 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3630 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3631 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3632 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3633 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3634
3635 // Trigger adapt up, expect increased fps (640x360@maxfps).
3636 video_stream_encoder_->TriggerQualityHigh();
3637 video_stream_encoder_->TriggerCpuUnderuse();
3638 timestamp_ms += kFrameIntervalMs;
3639 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3640 WaitForEncodedFrame(timestamp_ms);
3641 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3642 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3643 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3644 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3645 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3646 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3647 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3648
3649 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3650 video_stream_encoder_->TriggerQualityHigh();
3651 video_stream_encoder_->TriggerCpuUnderuse();
3652 timestamp_ms += kFrameIntervalMs;
3653 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3654 WaitForEncodedFrame(timestamp_ms);
3655 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3656 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3657 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3658 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3659 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3660 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3661 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3662
3663 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3664 video_stream_encoder_->TriggerQualityHigh();
3665 video_stream_encoder_->TriggerCpuUnderuse();
3666 timestamp_ms += kFrameIntervalMs;
3667 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3668 WaitForEncodedFrame(timestamp_ms);
3669 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3670 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3671 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3672 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3673 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3674 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3675 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3676
3677 video_stream_encoder_->Stop();
3678}
3679
3680TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003681 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003682 const int kWidth = 1280;
3683 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003684 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003685 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003686
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003687 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003688 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003689 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003690 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003691
3692 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003693 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003694 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003695 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3696 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3697
3698 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003699 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003700 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003701 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3702 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3703
mflodmancc3d4422017-08-03 08:27:51 -07003704 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003705}
3706
mflodmancc3d4422017-08-03 08:27:51 -07003707TEST_F(VideoStreamEncoderTest,
3708 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003709 const int kWidth = 1280;
3710 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003711 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003712 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003713
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003714 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003715 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003716 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003717 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003718
3719 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003720 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003721 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003722 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003723 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3724
3725 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003726 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003727 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003728 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003729 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3730
mflodmancc3d4422017-08-03 08:27:51 -07003731 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003732}
3733
mflodmancc3d4422017-08-03 08:27:51 -07003734TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003735 const int kWidth = 1280;
3736 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003737 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003738 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003739
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003740 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003741 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003742 video_stream_encoder_->SetSource(&source,
3743 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003744
3745 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3746 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003747 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003748 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3749 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3750 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3751
3752 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003753 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003754 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003755 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3756 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3757 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3758
mflodmancc3d4422017-08-03 08:27:51 -07003759 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003760}
3761
mflodmancc3d4422017-08-03 08:27:51 -07003762TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003763 const int kWidth = 1280;
3764 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003765 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003766 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003767
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003768 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003769 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003770 video_stream_encoder_->SetSource(&source,
3771 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003772
3773 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3774 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003775 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003776 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3777 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3778 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3779
3780 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003781 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003782 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003783 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3784 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3785 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3786
mflodmancc3d4422017-08-03 08:27:51 -07003787 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003788}
3789
mflodmancc3d4422017-08-03 08:27:51 -07003790TEST_F(VideoStreamEncoderTest,
3791 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003792 const int kWidth = 1280;
3793 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003794 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003795 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003796
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003797 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003798 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003799 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003800 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003801 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003802
3803 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003804 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003805 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003806 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3807 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3808
3809 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003810 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003811 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003812 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003813 EXPECT_THAT(source.sink_wants(),
3814 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003815 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3816 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3817
3818 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003819 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003820 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003821 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3822 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3823 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3824
mflodmancc3d4422017-08-03 08:27:51 -07003825 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003826}
3827
mflodmancc3d4422017-08-03 08:27:51 -07003828TEST_F(VideoStreamEncoderTest,
3829 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003830 const int kWidth = 1280;
3831 const int kHeight = 720;
3832 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003833 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003834 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003835
3836 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3837 stats.input_frame_rate = kInputFps;
3838 stats_proxy_->SetMockStats(stats);
3839
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003840 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003841 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3842 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003843 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003844
3845 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003846 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003847 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3848 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003849 EXPECT_THAT(video_source_.sink_wants(),
3850 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003851
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003852 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003853 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003854 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003855 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003856 // Give the encoder queue time to process the change in degradation preference
3857 // by waiting for an encoded frame.
3858 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3859 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003860 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003861
3862 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003863 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003864 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3865 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003866 EXPECT_THAT(new_video_source.sink_wants(),
3867 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003868
3869 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003870 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003871 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003872
mflodmancc3d4422017-08-03 08:27:51 -07003873 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003874}
3875
mflodmancc3d4422017-08-03 08:27:51 -07003876TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003877 const int kWidth = 1280;
3878 const int kHeight = 720;
3879 const size_t kNumFrames = 10;
3880
Henrik Boström381d1092020-05-12 18:49:07 +02003881 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003882 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003883
asaperssond0de2952017-04-21 01:47:31 -07003884 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003885 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003886 video_source_.set_adaptation_enabled(true);
3887
3888 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3889 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3890
3891 int downscales = 0;
3892 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003893 video_source_.IncomingCapturedFrame(
3894 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3895 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003896
asaperssonfab67072017-04-04 05:51:49 -07003897 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003898 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003899 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003900 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003901
3902 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3903 ++downscales;
3904
3905 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3906 EXPECT_EQ(downscales,
3907 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3908 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003909 }
mflodmancc3d4422017-08-03 08:27:51 -07003910 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003911}
3912
mflodmancc3d4422017-08-03 08:27:51 -07003913TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003914 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3915 const int kWidth = 1280;
3916 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003917 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003918 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003919
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003920 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003921 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003922 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003923 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003924 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003925
Åsa Persson8c1bf952018-09-13 10:42:19 +02003926 int64_t timestamp_ms = kFrameIntervalMs;
3927 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003928 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003929 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003930 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3931 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3932
3933 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003934 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003935 timestamp_ms += kFrameIntervalMs;
3936 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3937 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003938 EXPECT_THAT(source.sink_wants(),
3939 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003940 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3941 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3942
3943 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003944 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003945 timestamp_ms += kFrameIntervalMs;
3946 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003947 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003948 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003949 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3950 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3951
3952 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003953 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003954 timestamp_ms += kFrameIntervalMs;
3955 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3956 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003957 EXPECT_THAT(source.sink_wants(),
3958 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003959 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3960 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3961
3962 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003963 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003964 timestamp_ms += kFrameIntervalMs;
3965 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003966 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003967 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003968 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3969 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3970
mflodmancc3d4422017-08-03 08:27:51 -07003971 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003972}
3973
mflodmancc3d4422017-08-03 08:27:51 -07003974TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003975 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3976 const int kWidth = 1280;
3977 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003978 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003979 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003980
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003981 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003982 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003983 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003984 video_stream_encoder_->SetSource(&source,
3985 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003986
Åsa Persson8c1bf952018-09-13 10:42:19 +02003987 int64_t timestamp_ms = kFrameIntervalMs;
3988 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003989 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003990 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003991 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3992 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3993
3994 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003995 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003996 timestamp_ms += kFrameIntervalMs;
3997 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3998 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003999 EXPECT_THAT(source.sink_wants(),
4000 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004001 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4002 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4003
4004 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07004005 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004006 timestamp_ms += kFrameIntervalMs;
4007 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07004008 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004009 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004010 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4011 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4012
4013 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07004014 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004015 timestamp_ms += kFrameIntervalMs;
4016 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4017 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004018 EXPECT_THAT(source.sink_wants(),
4019 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004020 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4021 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4022
4023 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07004024 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004025 timestamp_ms += kFrameIntervalMs;
4026 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07004027 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004028 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004029 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4030 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4031
mflodmancc3d4422017-08-03 08:27:51 -07004032 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004033}
4034
Sergey Silkin41c650b2019-10-14 13:12:19 +02004035TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
4036 fake_encoder_.SetResolutionBitrateLimits(
4037 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4038
Henrik Boström381d1092020-05-12 18:49:07 +02004039 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004040 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4041 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4042 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4043 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004044
4045 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004046 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004047 source.set_adaptation_enabled(true);
4048 video_stream_encoder_->SetSource(
4049 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4050
4051 // Insert 720p frame.
4052 int64_t timestamp_ms = kFrameIntervalMs;
4053 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4054 WaitForEncodedFrame(1280, 720);
4055
4056 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02004057 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004058 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4059 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4060 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4061 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004062 video_stream_encoder_->TriggerQualityLow();
4063
4064 // Insert 720p frame. It should be downscaled and encoded.
4065 timestamp_ms += kFrameIntervalMs;
4066 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4067 WaitForEncodedFrame(960, 540);
4068
4069 // Trigger adapt up. Higher resolution should not be requested duo to lack
4070 // of bitrate.
4071 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004072 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02004073
4074 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02004075 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004076 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4077 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4078 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4079 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004080
4081 // Trigger adapt up. Higher resolution should be requested.
4082 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004083 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02004084
4085 video_stream_encoder_->Stop();
4086}
4087
4088TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
4089 fake_encoder_.SetResolutionBitrateLimits(
4090 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4091
4092 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02004093 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004094 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4095 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4096 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4097 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004098
4099 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004100 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004101 source.set_adaptation_enabled(true);
4102 video_stream_encoder_->SetSource(
4103 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4104
4105 // Insert 720p frame. It should be dropped and lower resolution should be
4106 // requested.
4107 int64_t timestamp_ms = kFrameIntervalMs;
4108 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4109 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004110 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004111
4112 // Insert 720p frame. It should be downscaled and encoded.
4113 timestamp_ms += kFrameIntervalMs;
4114 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4115 WaitForEncodedFrame(960, 540);
4116
4117 video_stream_encoder_->Stop();
4118}
4119
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004120class BalancedDegradationTest : public VideoStreamEncoderTest {
4121 protected:
4122 void SetupTest() {
4123 // Reset encoder for field trials to take effect.
4124 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02004125 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004126
4127 // Enable BALANCED preference.
4128 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02004129 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4130 }
4131
Asa Persson606d3cb2021-10-04 10:07:11 +02004132 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02004133 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004134 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004135 }
4136
Åsa Persson45b176f2019-09-30 11:19:05 +02004137 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004138 timestamp_ms_ += kFrameIntervalMs;
4139 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02004140 }
4141
4142 void InsertFrameAndWaitForEncoded() {
4143 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004144 sink_.WaitForEncodedFrame(timestamp_ms_);
4145 }
4146
4147 const int kWidth = 640; // pixels:640x360=230400
4148 const int kHeight = 360;
4149 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4150 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004151 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004152};
4153
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004154TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004155 test::ScopedKeyValueConfig field_trials(
4156 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004157 "WebRTC-Video-BalancedDegradationSettings/"
4158 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4159 SetupTest();
4160
4161 // Force input frame rate.
4162 const int kInputFps = 24;
4163 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4164 stats.input_frame_rate = kInputFps;
4165 stats_proxy_->SetMockStats(stats);
4166
Åsa Persson45b176f2019-09-30 11:19:05 +02004167 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004168 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004169
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004170 // Trigger adapt down, expect scaled down framerate and resolution,
4171 // since Fps diff (input-requested:0) < threshold.
4172 video_stream_encoder_->TriggerQualityLow();
4173 EXPECT_THAT(source_.sink_wants(),
4174 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004175
4176 video_stream_encoder_->Stop();
4177}
4178
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004179TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004180 test::ScopedKeyValueConfig field_trials(
4181 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004182 "WebRTC-Video-BalancedDegradationSettings/"
4183 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4184 SetupTest();
4185
4186 // Force input frame rate.
4187 const int kInputFps = 25;
4188 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4189 stats.input_frame_rate = kInputFps;
4190 stats_proxy_->SetMockStats(stats);
4191
Åsa Persson45b176f2019-09-30 11:19:05 +02004192 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004193 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004194
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004195 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4196 // Fps diff (input-requested:1) == threshold.
4197 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004198 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004199
4200 video_stream_encoder_->Stop();
4201}
4202
4203TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004204 test::ScopedKeyValueConfig field_trials(
4205 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004206 "WebRTC-Video-BalancedDegradationSettings/"
4207 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4208 SetupTest();
4209
4210 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4211
Åsa Persson45b176f2019-09-30 11:19:05 +02004212 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004213 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004214
4215 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4216 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004217 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004218
4219 video_stream_encoder_->Stop();
4220}
4221
Åsa Perssonccfb3402019-09-25 15:13:04 +02004222TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004223 test::ScopedKeyValueConfig field_trials(
4224 field_trials_,
Åsa Persson1b247f12019-08-14 17:26:39 +02004225 "WebRTC-Video-BalancedDegradationSettings/"
4226 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004227 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004228
Asa Persson606d3cb2021-10-04 10:07:11 +02004229 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4230 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4231 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004232
Åsa Persson45b176f2019-09-30 11:19:05 +02004233 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004234 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004235 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4236
4237 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4238 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004239 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004240 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004241 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4242
4243 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4244 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004245 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004246 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004247 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4248
Åsa Persson30ab0152019-08-27 12:22:33 +02004249 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4250 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004251 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004252 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004253 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004254 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4255
4256 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004257 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004258 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004259 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004260
Åsa Persson30ab0152019-08-27 12:22:33 +02004261 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004262 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004263 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004264 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004265 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004266 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4267
4268 video_stream_encoder_->Stop();
4269}
4270
Åsa Perssonccfb3402019-09-25 15:13:04 +02004271TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004272 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004273 test::ScopedKeyValueConfig field_trials(
4274 field_trials_,
Åsa Persson45b176f2019-09-30 11:19:05 +02004275 "WebRTC-Video-BalancedDegradationSettings/"
4276 "pixels:57600|129600|230400,fps:7|24|24/");
4277 SetupTest();
Asa Persson606d3cb2021-10-04 10:07:11 +02004278 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 11:19:05 +02004279
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004280 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004281
4282 // Insert frame, expect scaled down:
4283 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4284 InsertFrame();
4285 EXPECT_FALSE(WaitForFrame(1000));
4286 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4287 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4288
4289 // Insert frame, expect scaled down:
4290 // resolution (320x180@24fps).
4291 InsertFrame();
4292 EXPECT_FALSE(WaitForFrame(1000));
4293 EXPECT_LT(source_.sink_wants().max_pixel_count,
4294 source_.last_wants().max_pixel_count);
4295 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4296
4297 // Frame should not be dropped (min pixels per frame reached).
4298 InsertFrameAndWaitForEncoded();
4299
4300 video_stream_encoder_->Stop();
4301}
4302
4303TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004304 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004305 test::ScopedKeyValueConfig field_trials(
4306 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004307 "WebRTC-Video-BalancedDegradationSettings/"
4308 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004309 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004310
Asa Persson606d3cb2021-10-04 10:07:11 +02004311 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4312 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4313 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004314
Åsa Persson45b176f2019-09-30 11:19:05 +02004315 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004316 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004317 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4318
4319 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4320 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004321 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004322 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004323 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4324
4325 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4326 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004327 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004328 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004329 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4330
4331 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4332 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004333 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004334 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004335 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4336
Åsa Persson30ab0152019-08-27 12:22:33 +02004337 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4338 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004339 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004340 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004341 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4342
4343 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4344 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004345 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004346 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4347
4348 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004349 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004350 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004351 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004352 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004353 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4354
4355 video_stream_encoder_->Stop();
4356}
4357
Åsa Perssonccfb3402019-09-25 15:13:04 +02004358TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004359 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004360 test::ScopedKeyValueConfig field_trials(
4361 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004362 "WebRTC-Video-BalancedDegradationSettings/"
4363 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004364 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004365
Asa Persson606d3cb2021-10-04 10:07:11 +02004366 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4367 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4368 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4369 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4370 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004371
Åsa Persson45b176f2019-09-30 11:19:05 +02004372 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004373 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004374 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4375
4376 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4377 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004378 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004379 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004380 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4381
4382 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4383 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004384 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004385 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004386 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4387
4388 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4389 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004390 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004391 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004392 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4393
4394 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4395 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004396 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004397 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4398
4399 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004400 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004401 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004402 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004403 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004404 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4405
4406 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004407 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004408 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004409 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004410 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4411
4412 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004413 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004414 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004415 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004416 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004417 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4418
Åsa Persson1b247f12019-08-14 17:26:39 +02004419 video_stream_encoder_->Stop();
4420}
4421
mflodmancc3d4422017-08-03 08:27:51 -07004422TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004423 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4424 const int kWidth = 1280;
4425 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004426 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004427 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004428
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004429 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004430 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004431 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004432 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004433 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004434
Åsa Persson8c1bf952018-09-13 10:42:19 +02004435 int64_t timestamp_ms = kFrameIntervalMs;
4436 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004437 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004438 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004439 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4440 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4441 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4442 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4443
4444 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004445 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004446 timestamp_ms += kFrameIntervalMs;
4447 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4448 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004449 EXPECT_THAT(source.sink_wants(),
4450 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004451 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4452 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4453 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4454 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4455
4456 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004457 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004458 timestamp_ms += kFrameIntervalMs;
4459 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4460 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004461 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004462 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4463 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4464 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4465 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4466
Jonathan Yubc771b72017-12-08 17:04:29 -08004467 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004468 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004469 timestamp_ms += kFrameIntervalMs;
4470 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4471 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004472 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004473 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4474 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004475 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004476 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4477
Jonathan Yubc771b72017-12-08 17:04:29 -08004478 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004479 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004480 timestamp_ms += kFrameIntervalMs;
4481 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4482 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004483 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004484 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004485 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4486 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4487 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4488 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4489
Jonathan Yubc771b72017-12-08 17:04:29 -08004490 // Trigger quality adapt down, expect no change (min resolution reached).
4491 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004492 timestamp_ms += kFrameIntervalMs;
4493 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4494 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004495 EXPECT_THAT(source.sink_wants(), FpsMax());
4496 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004497 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4498 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4499 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4500 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4501
Evan Shrubsole64469032020-06-11 10:45:29 +02004502 // Trigger quality adapt up, expect upscaled resolution (480x270).
4503 video_stream_encoder_->TriggerQualityHigh();
4504 timestamp_ms += kFrameIntervalMs;
4505 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4506 WaitForEncodedFrame(timestamp_ms);
4507 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4508 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4509 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4510 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4511 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4512
4513 // Trigger quality and cpu adapt up since both are most limited, expect
4514 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004515 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004516 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004517 timestamp_ms += kFrameIntervalMs;
4518 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4519 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004520 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004521 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4522 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4523 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004524 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004525
Evan Shrubsole64469032020-06-11 10:45:29 +02004526 // Trigger quality and cpu adapt up since both are most limited, expect
4527 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004528 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004529 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004530 timestamp_ms += kFrameIntervalMs;
4531 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4532 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004533 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004534 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004535 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004536 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004537 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4538 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004539
Evan Shrubsole64469032020-06-11 10:45:29 +02004540 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4541 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004542 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004543 timestamp_ms += kFrameIntervalMs;
4544 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4545 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004546 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004547 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4548 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004549 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004550 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004551
4552 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004553 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004554 timestamp_ms += kFrameIntervalMs;
4555 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004556 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004557 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004558 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004559 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4560 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004561 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004562 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004563
mflodmancc3d4422017-08-03 08:27:51 -07004564 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004565}
4566
mflodmancc3d4422017-08-03 08:27:51 -07004567TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004568 const int kWidth = 640;
4569 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004570
Henrik Boström381d1092020-05-12 18:49:07 +02004571 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004572 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004573
perkj803d97f2016-11-01 11:45:46 -07004574 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004575 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004576 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004577 }
4578
mflodmancc3d4422017-08-03 08:27:51 -07004579 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004580 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004581 video_source_.IncomingCapturedFrame(CreateFrame(
4582 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004583 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004584 }
4585
mflodmancc3d4422017-08-03 08:27:51 -07004586 video_stream_encoder_->Stop();
4587 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004588 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004589
Ying Wangef3998f2019-12-09 13:06:53 +01004590 EXPECT_METRIC_EQ(
4591 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4592 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004593 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4594}
4595
mflodmancc3d4422017-08-03 08:27:51 -07004596TEST_F(VideoStreamEncoderTest,
4597 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004598 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004599 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004600 const int kWidth = 640;
4601 const int kHeight = 360;
4602
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004603 video_stream_encoder_->SetSource(&video_source_,
4604 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004605
4606 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4607 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004608 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004609 }
4610
mflodmancc3d4422017-08-03 08:27:51 -07004611 video_stream_encoder_->Stop();
4612 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004613 stats_proxy_.reset();
4614
4615 EXPECT_EQ(0,
4616 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4617}
4618
Per Kjellanderdcef6412020-10-07 15:09:05 +02004619TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4620 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004621 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004622 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004623
4624 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004625 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 10:07:11 +02004626 SimulcastRateAllocator(fake_encoder_.config())
4627 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004628 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004629
Henrik Boström381d1092020-05-12 18:49:07 +02004630 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004631 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004632
sprang57c2fff2017-01-16 06:24:02 -08004633 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004634 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4635 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004636 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4637 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4638
Erik Språngd7329ca2019-02-21 21:19:53 +01004639 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004640 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004641 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004642
Per Kjellanderdcef6412020-10-07 15:09:05 +02004643 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004644 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004645 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4646 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004647 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004648 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004649
Per Kjellanderdcef6412020-10-07 15:09:05 +02004650 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004651 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004652 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004653 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004654 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4655 WaitForEncodedFrame(CurrentTimeMs());
4656 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004657 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004658 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004659
mflodmancc3d4422017-08-03 08:27:51 -07004660 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004661}
4662
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004663TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004664 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004665 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004666 kVideoLayersAllocation);
4667
4668 const int kDefaultFps = 30;
4669
4670 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004671 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004672
4673 video_source_.IncomingCapturedFrame(
4674 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4675 WaitForEncodedFrame(CurrentTimeMs());
4676 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4677 VideoLayersAllocation last_layer_allocation =
4678 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004679 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004680 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4681
4682 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004683 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004684 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 10:07:11 +02004685 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 17:53:22 +02004686 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4687
Erik Språng9d69cbe2020-10-22 17:44:42 +02004688 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004689 int number_of_layers_allocation = 1;
4690 const int64_t start_time_ms = CurrentTimeMs();
4691 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4692 video_source_.IncomingCapturedFrame(
4693 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4694 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004695 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4696 number_of_layers_allocation = sink_.number_of_layers_allocations();
4697 VideoLayersAllocation new_allocation =
4698 sink_.GetLastVideoLayersAllocation();
4699 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4700 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4701 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4702 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4703 .target_bitrate_per_temporal_layer,
4704 last_layer_allocation.active_spatial_layers[0]
4705 .target_bitrate_per_temporal_layer);
4706 last_layer_allocation = new_allocation;
4707 }
4708 }
4709 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4710 video_stream_encoder_->Stop();
4711}
4712
4713TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004714 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004715 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4716 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4717 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004718 VideoEncoderConfig video_encoder_config;
4719 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4720 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004721 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004722 video_encoder_config.content_type =
4723 VideoEncoderConfig::ContentType::kRealtimeVideo;
4724 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004725 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004726 VideoEncoder::GetDefaultVp8Settings());
4727 for (auto& layer : video_encoder_config.simulcast_layers) {
4728 layer.num_temporal_layers = 2;
4729 }
4730 // Simulcast layers are used for enabling/disabling streams.
4731 video_encoder_config.simulcast_layers[0].active = true;
4732 video_encoder_config.simulcast_layers[1].active = false;
4733 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004734 ConfigureEncoder(std::move(video_encoder_config),
4735 VideoStreamEncoder::BitrateAllocationCallbackType::
4736 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004737
4738 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004739 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004740
4741 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4742 WaitForEncodedFrame(CurrentTimeMs());
4743 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4744 VideoLayersAllocation last_layer_allocation =
4745 sink_.GetLastVideoLayersAllocation();
4746
4747 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4748 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4749 .target_bitrate_per_temporal_layer,
4750 SizeIs(2));
4751 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4752 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4753 video_stream_encoder_->Stop();
4754}
4755
4756TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004757 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004758 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4759 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4760 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004761 VideoEncoderConfig video_encoder_config;
4762 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4763 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004764 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004765 video_encoder_config.content_type =
4766 VideoEncoderConfig::ContentType::kRealtimeVideo;
4767 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004768 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004769 VideoEncoder::GetDefaultVp8Settings());
4770 for (auto& layer : video_encoder_config.simulcast_layers) {
4771 layer.num_temporal_layers = 2;
4772 }
4773 // Simulcast layers are used for enabling/disabling streams.
4774 video_encoder_config.simulcast_layers[0].active = true;
4775 video_encoder_config.simulcast_layers[1].active = false;
4776 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004777 ConfigureEncoder(std::move(video_encoder_config),
4778 VideoStreamEncoder::BitrateAllocationCallbackType::
4779 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004780
4781 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004782 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004783
4784 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4785 WaitForEncodedFrame(CurrentTimeMs());
4786 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4787 VideoLayersAllocation last_layer_allocation =
4788 sink_.GetLastVideoLayersAllocation();
4789
4790 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4791 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4792 .target_bitrate_per_temporal_layer,
4793 SizeIs(2));
4794 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4795
4796 video_stream_encoder_->Stop();
4797}
4798
4799TEST_F(VideoStreamEncoderTest,
4800 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4801 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4802 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004803 VideoEncoderConfig video_encoder_config;
4804 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4805 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004806 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004807 video_encoder_config.content_type =
4808 VideoEncoderConfig::ContentType::kRealtimeVideo;
4809 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4810 vp9_settings.numberOfSpatialLayers = 2;
4811 vp9_settings.numberOfTemporalLayers = 2;
4812 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4813 vp9_settings.automaticResizeOn = false;
4814 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004815 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004816 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004817 ConfigureEncoder(std::move(video_encoder_config),
4818 VideoStreamEncoder::BitrateAllocationCallbackType::
4819 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004820
4821 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004822 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004823
4824 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4825 WaitForEncodedFrame(CurrentTimeMs());
4826 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4827 VideoLayersAllocation last_layer_allocation =
4828 sink_.GetLastVideoLayersAllocation();
4829
4830 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4831 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4832 .target_bitrate_per_temporal_layer,
4833 SizeIs(2));
4834 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4835 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4836 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4837 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4838 .target_bitrate_per_temporal_layer,
4839 SizeIs(2));
4840 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4841 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4842 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4843
4844 // Since full SVC is used, expect the top layer to utilize the full target
4845 // rate.
4846 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4847 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004848 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004849 video_stream_encoder_->Stop();
4850}
4851
4852TEST_F(VideoStreamEncoderTest,
4853 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4854 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4855 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004856 VideoEncoderConfig video_encoder_config;
4857 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4858 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004859 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004860 video_encoder_config.content_type =
4861 VideoEncoderConfig::ContentType::kRealtimeVideo;
4862 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4863 vp9_settings.numberOfSpatialLayers = 2;
4864 vp9_settings.numberOfTemporalLayers = 2;
4865 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4866 vp9_settings.automaticResizeOn = false;
4867 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004868 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004869 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004870 ConfigureEncoder(std::move(video_encoder_config),
4871 VideoStreamEncoder::BitrateAllocationCallbackType::
4872 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004873
4874 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004875 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004876
4877 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4878 WaitForEncodedFrame(CurrentTimeMs());
4879 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4880 VideoLayersAllocation last_layer_allocation =
4881 sink_.GetLastVideoLayersAllocation();
4882
4883 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4884 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4885 .target_bitrate_per_temporal_layer,
4886 SizeIs(1));
4887 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4888 .target_bitrate_per_temporal_layer,
4889 SizeIs(1));
4890 // Since full SVC is used, expect the top layer to utilize the full target
4891 // rate.
4892 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4893 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004894 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004895 video_stream_encoder_->Stop();
4896}
4897
4898TEST_F(VideoStreamEncoderTest,
4899 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4900 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4901 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004902 VideoEncoderConfig video_encoder_config;
4903 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4904 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004905 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004906 video_encoder_config.content_type =
4907 VideoEncoderConfig::ContentType::kRealtimeVideo;
4908 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4909 vp9_settings.numberOfSpatialLayers = 2;
4910 vp9_settings.numberOfTemporalLayers = 2;
4911 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4912 vp9_settings.automaticResizeOn = false;
4913 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004914 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004915 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004916 ConfigureEncoder(std::move(video_encoder_config),
4917 VideoStreamEncoder::BitrateAllocationCallbackType::
4918 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004919
4920 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004921 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004922
4923 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4924 WaitForEncodedFrame(CurrentTimeMs());
4925 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4926 VideoLayersAllocation last_layer_allocation =
4927 sink_.GetLastVideoLayersAllocation();
4928
4929 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4930 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4931 .target_bitrate_per_temporal_layer,
4932 SizeIs(2));
4933 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4934 .target_bitrate_per_temporal_layer,
4935 SizeIs(2));
4936 // Since KSVC is, spatial layers are independend except on key frames.
4937 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4938 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004939 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004940 video_stream_encoder_->Stop();
4941}
4942
4943TEST_F(VideoStreamEncoderTest,
4944 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4945 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4946 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4947 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004948 VideoEncoderConfig video_encoder_config;
4949 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4950 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004951 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004952 video_encoder_config.content_type =
4953 VideoEncoderConfig::ContentType::kRealtimeVideo;
4954 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4955 vp9_settings.numberOfSpatialLayers = 3;
4956 vp9_settings.numberOfTemporalLayers = 2;
4957 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4958 vp9_settings.automaticResizeOn = false;
4959 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004960 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004961 vp9_settings);
4962 // Simulcast layers are used for enabling/disabling streams.
4963 video_encoder_config.simulcast_layers.resize(3);
4964 video_encoder_config.simulcast_layers[0].active = false;
4965 video_encoder_config.simulcast_layers[1].active = true;
4966 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004967 ConfigureEncoder(std::move(video_encoder_config),
4968 VideoStreamEncoder::BitrateAllocationCallbackType::
4969 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004970
4971 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004972 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004973
4974 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4975 WaitForEncodedFrame(CurrentTimeMs());
4976 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4977 VideoLayersAllocation last_layer_allocation =
4978 sink_.GetLastVideoLayersAllocation();
4979
4980 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4981 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4982 .target_bitrate_per_temporal_layer,
4983 SizeIs(2));
4984 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4985 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4986
4987 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4988 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4989 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4990 .target_bitrate_per_temporal_layer,
4991 SizeIs(2));
4992 // Since full SVC is used, expect the top layer to utilize the full target
4993 // rate.
4994 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4995 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004996 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004997 video_stream_encoder_->Stop();
4998}
4999
5000TEST_F(VideoStreamEncoderTest,
5001 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
5002 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5003 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5004 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005005 VideoEncoderConfig video_encoder_config;
5006 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5007 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005008 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005009 video_encoder_config.content_type =
5010 VideoEncoderConfig::ContentType::kRealtimeVideo;
5011 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5012 vp9_settings.numberOfSpatialLayers = 3;
5013 vp9_settings.numberOfTemporalLayers = 2;
5014 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5015 vp9_settings.automaticResizeOn = false;
5016 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005017 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005018 vp9_settings);
5019 // Simulcast layers are used for enabling/disabling streams.
5020 video_encoder_config.simulcast_layers.resize(3);
5021 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005022 ConfigureEncoder(std::move(video_encoder_config),
5023 VideoStreamEncoder::BitrateAllocationCallbackType::
5024 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005025
5026 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005027 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005028
5029 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5030 WaitForEncodedFrame(CurrentTimeMs());
5031 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5032 VideoLayersAllocation last_layer_allocation =
5033 sink_.GetLastVideoLayersAllocation();
5034
5035 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
5036 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5037 .target_bitrate_per_temporal_layer,
5038 SizeIs(2));
5039 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
5040 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5041
5042 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
5043 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
5044 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
5045 .target_bitrate_per_temporal_layer,
5046 SizeIs(2));
5047 video_stream_encoder_->Stop();
5048}
5049
5050TEST_F(VideoStreamEncoderTest,
5051 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
5052 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5053 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5054 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005055 VideoEncoderConfig video_encoder_config;
5056 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5057 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005058 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005059 video_encoder_config.content_type =
5060 VideoEncoderConfig::ContentType::kRealtimeVideo;
5061 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5062 vp9_settings.numberOfSpatialLayers = 3;
5063 vp9_settings.numberOfTemporalLayers = 2;
5064 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5065 vp9_settings.automaticResizeOn = false;
5066 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005067 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005068 vp9_settings);
5069 // Simulcast layers are used for enabling/disabling streams.
5070 video_encoder_config.simulcast_layers.resize(3);
5071 video_encoder_config.simulcast_layers[0].active = false;
5072 video_encoder_config.simulcast_layers[1].active = false;
5073 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005074 ConfigureEncoder(std::move(video_encoder_config),
5075 VideoStreamEncoder::BitrateAllocationCallbackType::
5076 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005077
5078 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005079 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005080
5081 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5082 WaitForEncodedFrame(CurrentTimeMs());
5083 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5084 VideoLayersAllocation last_layer_allocation =
5085 sink_.GetLastVideoLayersAllocation();
5086
5087 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5088 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5089 .target_bitrate_per_temporal_layer,
5090 SizeIs(2));
5091 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5092 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5093 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5094 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02005095 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005096 video_stream_encoder_->Stop();
5097}
5098
5099TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5100 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005101 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005102 kVideoLayersAllocation);
5103 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005104 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005105
5106 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5107 WaitForEncodedFrame(CurrentTimeMs());
5108 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5109 VideoLayersAllocation last_layer_allocation =
5110 sink_.GetLastVideoLayersAllocation();
5111
5112 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5113 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5114 .target_bitrate_per_temporal_layer,
5115 SizeIs(1));
5116 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5117 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005118 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005119 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5120 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5121 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5122 video_stream_encoder_->Stop();
5123}
5124
5125TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02005126 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5127 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005128 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02005129 kVideoLayersAllocation);
5130
5131 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005132 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005133
5134 video_source_.IncomingCapturedFrame(
5135 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5136 WaitForEncodedFrame(CurrentTimeMs());
5137 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5138 VideoLayersAllocation last_layer_allocation =
5139 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02005140 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02005141 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5142 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5143 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005144 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 17:53:22 +02005145
5146 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005147 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5148 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005149 video_source_.IncomingCapturedFrame(
5150 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5151 WaitForEncodedFrame(CurrentTimeMs());
5152
5153 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5154 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5155 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5156 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5157 .target_bitrate_per_temporal_layer[0],
5158 DataRate::Zero());
5159
5160 video_stream_encoder_->Stop();
5161}
5162
Per Kjellander4190ce92020-12-15 17:24:55 +01005163TEST_F(VideoStreamEncoderTest,
5164 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5165 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005166 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01005167 kVideoLayersAllocation);
5168
5169 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005170 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5171 0, 0, 0);
Per Kjellander4190ce92020-12-15 17:24:55 +01005172
5173 video_source_.IncomingCapturedFrame(
5174 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5175 WaitForEncodedFrame(CurrentTimeMs());
5176 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5177 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5178 SizeIs(2));
5179 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5180 codec_width_);
5181 EXPECT_EQ(
5182 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5183 codec_height_);
5184
5185 video_source_.IncomingCapturedFrame(
5186 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5187 WaitForEncodedFrame(CurrentTimeMs());
5188 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5189 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5190 SizeIs(2));
5191 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5192 codec_width_ / 2);
5193 EXPECT_EQ(
5194 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5195 codec_height_ / 2);
5196
5197 video_stream_encoder_->Stop();
5198}
5199
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005200TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5201 // 2 TLs configured, temporal layers supported by encoder.
5202 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005203 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005204 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005205 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005206 fake_encoder_.SetTemporalLayersSupported(0, true);
5207
5208 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 10:07:11 +02005209 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005210 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005211 kNumTemporalLayers, /*temporal_id*/ 0,
5212 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005213 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005214 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005215 kNumTemporalLayers, /*temporal_id*/ 1,
5216 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005217 VideoBitrateAllocation expected_bitrate;
5218 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5219 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5220
5221 VerifyAllocatedBitrate(expected_bitrate);
5222 video_stream_encoder_->Stop();
5223}
5224
5225TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5226 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005227 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005228 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005229 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005230 fake_encoder_.SetTemporalLayersSupported(0, false);
5231
5232 // Temporal layers not supported by the encoder.
5233 // Total bitrate should be at ti:0.
5234 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005235 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005236
5237 VerifyAllocatedBitrate(expected_bitrate);
5238 video_stream_encoder_->Stop();
5239}
5240
5241TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005242 webrtc::test::ScopedKeyValueConfig field_trials(
5243 field_trials_,
Per Kjellanderdcef6412020-10-07 15:09:05 +02005244 "WebRTC-Video-QualityScalerSettings/"
5245 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5246 // Reset encoder for field trials to take effect.
5247 ConfigureEncoder(video_encoder_config_.Copy());
5248
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005249 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005250 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005251 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005252 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005253 fake_encoder_.SetTemporalLayersSupported(0, true);
5254 fake_encoder_.SetTemporalLayersSupported(1, false);
5255
5256 const int kS0Bps = 150000;
5257 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005258 kS0Bps *
5259 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5260 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005261 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005262 kS0Bps *
5263 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5264 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005265 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005266 // Temporal layers not supported by si:1.
5267 VideoBitrateAllocation expected_bitrate;
5268 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5269 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5270 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5271
5272 VerifyAllocatedBitrate(expected_bitrate);
5273 video_stream_encoder_->Stop();
5274}
5275
Niels Möller7dc26b72017-12-06 10:27:48 +01005276TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5277 const int kFrameWidth = 1280;
5278 const int kFrameHeight = 720;
5279 const int kFramerate = 24;
5280
Henrik Boström381d1092020-05-12 18:49:07 +02005281 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005282 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005283 test::FrameForwarder source;
5284 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005285 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005286
5287 // Insert a single frame, triggering initial configuration.
5288 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5289 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5290
5291 EXPECT_EQ(
5292 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5293 kDefaultFramerate);
5294
5295 // Trigger reconfigure encoder (without resetting the entire instance).
5296 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005297 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5298 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005299 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005300 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005301 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005302 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5303
5304 // Detector should be updated with fps limit from codec config.
5305 EXPECT_EQ(
5306 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5307 kFramerate);
5308
5309 // Trigger overuse, max framerate should be reduced.
5310 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5311 stats.input_frame_rate = kFramerate;
5312 stats_proxy_->SetMockStats(stats);
5313 video_stream_encoder_->TriggerCpuOveruse();
5314 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5315 int adapted_framerate =
5316 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5317 EXPECT_LT(adapted_framerate, kFramerate);
5318
5319 // Trigger underuse, max framerate should go back to codec configured fps.
5320 // Set extra low fps, to make sure it's actually reset, not just incremented.
5321 stats = stats_proxy_->GetStats();
5322 stats.input_frame_rate = adapted_framerate / 2;
5323 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005324 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005325 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5326 EXPECT_EQ(
5327 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5328 kFramerate);
5329
5330 video_stream_encoder_->Stop();
5331}
5332
5333TEST_F(VideoStreamEncoderTest,
5334 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5335 const int kFrameWidth = 1280;
5336 const int kFrameHeight = 720;
5337 const int kLowFramerate = 15;
5338 const int kHighFramerate = 25;
5339
Henrik Boström381d1092020-05-12 18:49:07 +02005340 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005341 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005342 test::FrameForwarder source;
5343 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005344 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005345
5346 // Trigger initial configuration.
5347 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005348 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5349 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005350 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005351 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005352 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005353 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005354 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5355
5356 EXPECT_EQ(
5357 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5358 kLowFramerate);
5359
5360 // Trigger overuse, max framerate should be reduced.
5361 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5362 stats.input_frame_rate = kLowFramerate;
5363 stats_proxy_->SetMockStats(stats);
5364 video_stream_encoder_->TriggerCpuOveruse();
5365 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5366 int adapted_framerate =
5367 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5368 EXPECT_LT(adapted_framerate, kLowFramerate);
5369
5370 // Reconfigure the encoder with a new (higher max framerate), max fps should
5371 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005372 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005373 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5374 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005375 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005376 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5377
5378 EXPECT_EQ(
5379 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5380 adapted_framerate);
5381
5382 // Trigger underuse, max framerate should go back to codec configured fps.
5383 stats = stats_proxy_->GetStats();
5384 stats.input_frame_rate = adapted_framerate;
5385 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005386 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005387 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5388 EXPECT_EQ(
5389 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5390 kHighFramerate);
5391
5392 video_stream_encoder_->Stop();
5393}
5394
mflodmancc3d4422017-08-03 08:27:51 -07005395TEST_F(VideoStreamEncoderTest,
5396 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005397 const int kFrameWidth = 1280;
5398 const int kFrameHeight = 720;
5399 const int kFramerate = 24;
5400
Henrik Boström381d1092020-05-12 18:49:07 +02005401 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005402 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005403 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005404 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005405 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005406
5407 // Trigger initial configuration.
5408 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005409 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5410 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005411 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 04:21:07 -07005412 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005413 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005414 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005415 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005416
Niels Möller7dc26b72017-12-06 10:27:48 +01005417 EXPECT_EQ(
5418 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5419 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005420
5421 // Trigger overuse, max framerate should be reduced.
5422 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5423 stats.input_frame_rate = kFramerate;
5424 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005425 video_stream_encoder_->TriggerCpuOveruse();
5426 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005427 int adapted_framerate =
5428 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005429 EXPECT_LT(adapted_framerate, kFramerate);
5430
5431 // Change degradation preference to not enable framerate scaling. Target
5432 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005433 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005434 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005435 EXPECT_EQ(
5436 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5437 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005438
mflodmancc3d4422017-08-03 08:27:51 -07005439 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005440}
5441
mflodmancc3d4422017-08-03 08:27:51 -07005442TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005443 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005444 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005445 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5446 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5447 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005448 const int kWidth = 640;
5449 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005450
asaperssonfab67072017-04-04 05:51:49 -07005451 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005452
5453 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005454 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005455
5456 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005457 EXPECT_TRUE_WAIT(
5458 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005459
sprangc5d62e22017-04-02 23:53:04 -07005460 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005461
asaperssonfab67072017-04-04 05:51:49 -07005462 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005463 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005464 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005465
5466 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005467 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005468
Henrik Boström2671dac2020-05-19 16:29:09 +02005469 EXPECT_TRUE_WAIT(
5470 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005471
mflodmancc3d4422017-08-03 08:27:51 -07005472 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005473}
5474
mflodmancc3d4422017-08-03 08:27:51 -07005475TEST_F(VideoStreamEncoderTest,
5476 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005477 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005478 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005479 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5480 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5481 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005482 const int kWidth = 640;
5483 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005484
5485 // We expect the n initial frames to get dropped.
5486 int i;
5487 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005488 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005489 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005490 }
5491 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005492 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005493 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005494
5495 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005496 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005497
mflodmancc3d4422017-08-03 08:27:51 -07005498 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005499}
5500
mflodmancc3d4422017-08-03 08:27:51 -07005501TEST_F(VideoStreamEncoderTest,
5502 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005503 const int kWidth = 640;
5504 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005505 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005506 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005507
5508 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005509 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005510 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005511
asaperssonfab67072017-04-04 05:51:49 -07005512 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005513 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005514 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005515
mflodmancc3d4422017-08-03 08:27:51 -07005516 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005517}
5518
mflodmancc3d4422017-08-03 08:27:51 -07005519TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005520 const int kWidth = 640;
5521 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005522 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005523
5524 VideoEncoderConfig video_encoder_config;
5525 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5526 // Make format different, to force recreation of encoder.
5527 video_encoder_config.video_format.parameters["foo"] = "foo";
5528 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005529 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005530 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005531 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005532
kthelgasonb83797b2017-02-14 11:57:25 -08005533 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005534 video_stream_encoder_->SetSource(&video_source_,
5535 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005536
asaperssonfab67072017-04-04 05:51:49 -07005537 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005538 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005539 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005540
mflodmancc3d4422017-08-03 08:27:51 -07005541 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005542 fake_encoder_.SetQualityScaling(true);
5543}
5544
Åsa Persson139f4dc2019-08-02 09:29:58 +02005545TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005546 webrtc::test::ScopedKeyValueConfig field_trials(
5547 field_trials_,
Åsa Persson139f4dc2019-08-02 09:29:58 +02005548 "WebRTC-Video-QualityScalerSettings/"
5549 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5550 // Reset encoder for field trials to take effect.
5551 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005552 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5553 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 09:29:58 +02005554 const int kWidth = 640;
5555 const int kHeight = 360;
5556
Henrik Boström381d1092020-05-12 18:49:07 +02005557 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005558 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005559 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5560 // Frame should not be dropped.
5561 WaitForEncodedFrame(1);
5562
Henrik Boström381d1092020-05-12 18:49:07 +02005563 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005564 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5565 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5566 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005567 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5568 // Frame should not be dropped.
5569 WaitForEncodedFrame(2);
5570
Henrik Boström381d1092020-05-12 18:49:07 +02005571 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005572 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5573 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5574 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005575 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5576 // Expect to drop this frame, the wait should time out.
5577 ExpectDroppedFrame();
5578
5579 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005580 EXPECT_TRUE_WAIT(
5581 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005582 video_stream_encoder_->Stop();
5583}
5584
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005585TEST_F(VideoStreamEncoderTest,
5586 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005587 webrtc::test::ScopedKeyValueConfig field_trials(
5588 field_trials_,
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005589 "WebRTC-Video-QualityScalerSettings/"
5590 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5591 fake_encoder_.SetQualityScaling(false);
5592 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005593 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5594 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005595 const int kWidth = 640;
5596 const int kHeight = 360;
5597
5598 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005599 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005600 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5601 // Frame should not be dropped.
5602 WaitForEncodedFrame(1);
5603
5604 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5605 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5606 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5607 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5608 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5609 // Frame should not be dropped.
5610 WaitForEncodedFrame(2);
5611
5612 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5613 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5614 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5615 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5616 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5617 // Not dropped since quality scaling is disabled.
5618 WaitForEncodedFrame(3);
5619
5620 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005621 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005622 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5623
5624 video_stream_encoder_->Stop();
5625}
5626
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005627TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005628 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005629 // Set simulcast.
5630 ResetEncoder("VP8", 3, 1, 1, false);
5631 fake_encoder_.SetQualityScaling(true);
5632 const int kWidth = 1280;
5633 const int kHeight = 720;
5634 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005635 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005636 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5637 // Frame should not be dropped.
5638 WaitForEncodedFrame(1);
5639
5640 // Trigger QVGA "singlecast"
5641 // Update the config.
5642 VideoEncoderConfig video_encoder_config;
5643 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5644 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005645 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005646 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005647 "VP8", /*max qp*/ 56, /*screencast*/ false,
5648 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005649 for (auto& layer : video_encoder_config.simulcast_layers) {
5650 layer.num_temporal_layers = 1;
5651 layer.max_framerate = kDefaultFramerate;
5652 }
Asa Persson606d3cb2021-10-04 10:07:11 +02005653 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005654 video_encoder_config.content_type =
5655 VideoEncoderConfig::ContentType::kRealtimeVideo;
5656
5657 video_encoder_config.simulcast_layers[0].active = true;
5658 video_encoder_config.simulcast_layers[1].active = false;
5659 video_encoder_config.simulcast_layers[2].active = false;
5660
5661 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5662 kMaxPayloadLength);
5663 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5664
5665 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5666 // Frame should not be dropped.
5667 WaitForEncodedFrame(2);
5668
5669 // Trigger HD "singlecast"
5670 video_encoder_config.simulcast_layers[0].active = false;
5671 video_encoder_config.simulcast_layers[1].active = false;
5672 video_encoder_config.simulcast_layers[2].active = true;
5673
5674 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5675 kMaxPayloadLength);
5676 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5677
5678 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5679 // Frame should be dropped because of initial frame drop.
5680 ExpectDroppedFrame();
5681
5682 // Expect the sink_wants to specify a scaled frame.
5683 EXPECT_TRUE_WAIT(
5684 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5685 video_stream_encoder_->Stop();
5686}
5687
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005688TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005689 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005690 // Set simulcast.
5691 ResetEncoder("VP9", 1, 1, 3, false);
5692 fake_encoder_.SetQualityScaling(true);
5693 const int kWidth = 1280;
5694 const int kHeight = 720;
5695 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005696 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005697 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5698 // Frame should not be dropped.
5699 WaitForEncodedFrame(1);
5700
5701 // Trigger QVGA "singlecast"
5702 // Update the config.
5703 VideoEncoderConfig video_encoder_config;
5704 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5705 &video_encoder_config);
5706 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5707 vp9_settings.numberOfSpatialLayers = 3;
5708 // Since only one layer is active - automatic resize should be enabled.
5709 vp9_settings.automaticResizeOn = true;
5710 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005711 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005712 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005713 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005714 video_encoder_config.content_type =
5715 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 16:22:11 +02005716 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005717 // which SVC layers are active.
5718 video_encoder_config.simulcast_layers.resize(3);
5719
5720 video_encoder_config.simulcast_layers[0].active = true;
5721 video_encoder_config.simulcast_layers[1].active = false;
5722 video_encoder_config.simulcast_layers[2].active = false;
5723
5724 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5725 kMaxPayloadLength);
5726 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5727
5728 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5729 // Frame should not be dropped.
5730 WaitForEncodedFrame(2);
5731
5732 // Trigger HD "singlecast"
5733 video_encoder_config.simulcast_layers[0].active = false;
5734 video_encoder_config.simulcast_layers[1].active = false;
5735 video_encoder_config.simulcast_layers[2].active = true;
5736
5737 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5738 kMaxPayloadLength);
5739 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5740
5741 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5742 // Frame should be dropped because of initial frame drop.
5743 ExpectDroppedFrame();
5744
5745 // Expect the sink_wants to specify a scaled frame.
5746 EXPECT_TRUE_WAIT(
5747 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5748 video_stream_encoder_->Stop();
5749}
5750
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005751TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005752 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5753 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5754 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5755 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5756 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5757 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5758 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5759 fake_encoder_.SetResolutionBitrateLimits(
5760 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5761
5762 VideoEncoderConfig video_encoder_config;
5763 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5764 &video_encoder_config);
5765 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5766 vp9_settings.numberOfSpatialLayers = 3;
5767 // Since only one layer is active - automatic resize should be enabled.
5768 vp9_settings.automaticResizeOn = true;
5769 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005770 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005771 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005772 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005773 video_encoder_config.content_type =
5774 VideoEncoderConfig::ContentType::kRealtimeVideo;
5775 // Simulcast layers are used to indicate which spatial layers are active.
5776 video_encoder_config.simulcast_layers.resize(3);
5777 video_encoder_config.simulcast_layers[0].active = false;
5778 video_encoder_config.simulcast_layers[1].active = true;
5779 video_encoder_config.simulcast_layers[2].active = false;
5780
5781 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5782 kMaxPayloadLength);
5783 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5784
5785 // The encoder bitrate limits for 360p should be used.
5786 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5787 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005788 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5789 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5790 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5791 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5792 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5793 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005794 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005795 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005796 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005797 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005798
5799 // The encoder bitrate limits for 270p should be used.
5800 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5801 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005802 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5803 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5804 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5805 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5806 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5807 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005808 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005809 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005810 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005811 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005812
5813 video_stream_encoder_->Stop();
5814}
5815
5816TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005817 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5818 VideoEncoderConfig video_encoder_config;
5819 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5820 &video_encoder_config);
5821 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5822 vp9_settings.numberOfSpatialLayers = 3;
5823 // Since only one layer is active - automatic resize should be enabled.
5824 vp9_settings.automaticResizeOn = true;
5825 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005826 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005827 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005828 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005829 video_encoder_config.content_type =
5830 VideoEncoderConfig::ContentType::kRealtimeVideo;
5831 // Simulcast layers are used to indicate which spatial layers are active.
5832 video_encoder_config.simulcast_layers.resize(3);
5833 video_encoder_config.simulcast_layers[0].active = false;
5834 video_encoder_config.simulcast_layers[1].active = true;
5835 video_encoder_config.simulcast_layers[2].active = false;
5836
5837 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5838 kMaxPayloadLength);
5839 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5840
5841 // The default bitrate limits for 360p should be used.
5842 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005843 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5844 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005845 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5846 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005847 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5848 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5849 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5850 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5851 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5852 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005853 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005854 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005855 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005856 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005857
5858 // The default bitrate limits for 270p should be used.
5859 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005860 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5861 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005862 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5863 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005864 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5865 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5866 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5867 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5868 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5869 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005870 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005871 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005872 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005873 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005874
5875 video_stream_encoder_->Stop();
5876}
5877
5878TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005879 webrtc::test::ScopedKeyValueConfig field_trials(
5880 field_trials_, "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
Åsa Persson258e9892021-02-25 10:39:51 +01005881 VideoEncoderConfig video_encoder_config;
5882 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5883 &video_encoder_config);
5884 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5885 vp9_settings.numberOfSpatialLayers = 3;
5886 // Since only one layer is active - automatic resize should be enabled.
5887 vp9_settings.automaticResizeOn = true;
5888 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005889 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005890 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005891 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005892 video_encoder_config.content_type =
5893 VideoEncoderConfig::ContentType::kRealtimeVideo;
5894 // Simulcast layers are used to indicate which spatial layers are active.
5895 video_encoder_config.simulcast_layers.resize(3);
5896 video_encoder_config.simulcast_layers[0].active = false;
5897 video_encoder_config.simulcast_layers[1].active = true;
5898 video_encoder_config.simulcast_layers[2].active = false;
5899
5900 // Reset encoder for field trials to take effect.
5901 ConfigureEncoder(video_encoder_config.Copy());
5902
5903 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5904 kMaxPayloadLength);
5905 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5906
5907 // The default bitrate limits for 360p should not be used.
5908 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005909 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5910 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005911 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5912 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005913 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5914 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5915 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5916 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5917 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5918 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005919 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005920 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005921
5922 video_stream_encoder_->Stop();
5923}
5924
5925TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5926 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5927 /*num_spatial_layers=*/1, /*screenshare=*/false);
5928
5929 // The default singlecast bitrate limits for 720p should not be used.
5930 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005931 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5932 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005933 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5934 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005935 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5936 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5937 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5938 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5939 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5940 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005941 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005942 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005943
5944 video_stream_encoder_->Stop();
5945}
5946
5947TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005948 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5949 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5950 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5951 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5952 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5953 fake_encoder_.SetResolutionBitrateLimits(
5954 {kEncoderLimits180p, kEncoderLimits720p});
5955
5956 VideoEncoderConfig video_encoder_config;
5957 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5958 &video_encoder_config);
5959 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5960 vp9_settings.numberOfSpatialLayers = 3;
5961 // Since only one layer is active - automatic resize should be enabled.
5962 vp9_settings.automaticResizeOn = true;
5963 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005964 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005965 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005966 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005967 video_encoder_config.content_type =
5968 VideoEncoderConfig::ContentType::kRealtimeVideo;
5969 // Simulcast layers are used to indicate which spatial layers are active.
5970 video_encoder_config.simulcast_layers.resize(3);
5971 video_encoder_config.simulcast_layers[0].active = true;
5972 video_encoder_config.simulcast_layers[1].active = false;
5973 video_encoder_config.simulcast_layers[2].active = false;
5974
5975 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5976 kMaxPayloadLength);
5977 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5978
5979 // Limits not applied on lowest stream, limits for 180p should not be used.
5980 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5981 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005982 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5983 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5984 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5985 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5986 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5987 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005988 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005989 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005990 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005991 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005992
5993 video_stream_encoder_->Stop();
5994}
5995
5996TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005997 InitialFrameDropActivatesWhenResolutionIncreases) {
5998 const int kWidth = 640;
5999 const int kHeight = 360;
6000
6001 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006002 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006003 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
6004 // Frame should not be dropped.
6005 WaitForEncodedFrame(1);
6006
6007 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006008 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006009 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
6010 // Frame should not be dropped, bitrate not too low for frame.
6011 WaitForEncodedFrame(2);
6012
6013 // Incoming resolution increases.
6014 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
6015 // Expect to drop this frame, bitrate too low for frame.
6016 ExpectDroppedFrame();
6017
6018 // Expect the sink_wants to specify a scaled frame.
6019 EXPECT_TRUE_WAIT(
6020 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
6021 video_stream_encoder_->Stop();
6022}
6023
6024TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
6025 const int kWidth = 640;
6026 const int kHeight = 360;
6027 // So that quality scaling doesn't happen by itself.
6028 fake_encoder_.SetQp(kQpHigh);
6029
6030 AdaptingFrameForwarder source(&time_controller_);
6031 source.set_adaptation_enabled(true);
6032 video_stream_encoder_->SetSource(
6033 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
6034
6035 int timestamp = 1;
6036
6037 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006038 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006039 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6040 WaitForEncodedFrame(timestamp);
6041 timestamp += 9000;
6042 // Long pause to disable all first BWE drop logic.
6043 AdvanceTime(TimeDelta::Millis(1000));
6044
6045 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006046 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006047 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6048 // Not dropped frame, as initial frame drop is disabled by now.
6049 WaitForEncodedFrame(timestamp);
6050 timestamp += 9000;
6051 AdvanceTime(TimeDelta::Millis(100));
6052
6053 // Quality adaptation down.
6054 video_stream_encoder_->TriggerQualityLow();
6055
6056 // Adaptation has an effect.
6057 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6058 5000);
6059
6060 // Frame isn't dropped as initial frame dropper is disabled.
6061 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6062 WaitForEncodedFrame(timestamp);
6063 timestamp += 9000;
6064 AdvanceTime(TimeDelta::Millis(100));
6065
6066 // Quality adaptation up.
6067 video_stream_encoder_->TriggerQualityHigh();
6068
6069 // Adaptation has an effect.
6070 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
6071 5000);
6072
6073 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6074 // Frame should not be dropped, as initial framedropper is off.
6075 WaitForEncodedFrame(timestamp);
6076
6077 video_stream_encoder_->Stop();
6078}
6079
Åsa Persson7f354f82021-02-04 15:52:15 +01006080TEST_F(VideoStreamEncoderTest,
6081 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
6082 const int kMinStartBps360p = 222000;
6083 fake_encoder_.SetResolutionBitrateLimits(
6084 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6085 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6086 800000)});
6087
6088 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6089 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6090 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6091 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
6092 0, 0, 0);
6093 // Frame should not be dropped, bitrate not too low for frame.
6094 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6095 WaitForEncodedFrame(1);
6096
6097 // Incoming resolution increases, initial frame drop activates.
6098 // Frame should be dropped, link allocation too low for frame.
6099 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6100 ExpectDroppedFrame();
6101
6102 // Expect sink_wants to specify a scaled frame.
6103 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6104 5000);
6105 video_stream_encoder_->Stop();
6106}
6107
6108TEST_F(VideoStreamEncoderTest,
6109 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6110 const int kMinStartBps360p = 222000;
6111 fake_encoder_.SetResolutionBitrateLimits(
6112 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6113 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6114 800000)});
6115
6116 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6117 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6118 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6119 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6120 0, 0, 0);
6121 // Frame should not be dropped, bitrate not too low for frame.
6122 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6123 WaitForEncodedFrame(1);
6124
6125 // Incoming resolution increases, initial frame drop activates.
6126 // Frame should be dropped, link allocation not too low for frame.
6127 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6128 WaitForEncodedFrame(2);
6129
6130 video_stream_encoder_->Stop();
6131}
6132
Åsa Perssone644a032019-11-08 15:56:00 +01006133TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006134 webrtc::test::ScopedKeyValueConfig field_trials(
6135 field_trials_,
Åsa Persson06defc42021-09-10 15:28:48 +02006136 "WebRTC-Video-QualityRampupSettings/"
6137 "min_pixels:921600,min_duration_ms:2000/");
6138
6139 const int kWidth = 1280;
6140 const int kHeight = 720;
6141 const int kFps = 10;
6142 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006143
6144 // Reset encoder for field trials to take effect.
6145 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 10:07:11 +02006146 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 11:34:32 +02006147 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01006148 ConfigureEncoder(std::move(config));
6149 fake_encoder_.SetQp(kQpLow);
6150
6151 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006152 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01006153 source.set_adaptation_enabled(true);
6154 video_stream_encoder_->SetSource(&source,
6155 DegradationPreference::MAINTAIN_FRAMERATE);
6156
6157 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 10:07:11 +02006158 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 18:49:07 +02006159 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006160 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006161
6162 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 15:28:48 +02006163 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006164 int64_t timestamp_ms = kFrameIntervalMs;
6165 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6166 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02006167 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6168 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01006169
6170 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02006171 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6172 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006173
Artem Titovab30d722021-07-27 16:22:11 +02006174 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 15:28:48 +02006175 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 15:56:00 +01006176 for (size_t i = 1; i <= 10; i++) {
6177 timestamp_ms += kFrameIntervalMs;
6178 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6179 WaitForEncodedFrame(timestamp_ms);
6180 }
Åsa Persson06defc42021-09-10 15:28:48 +02006181
6182 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6183 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6184 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6185 timestamp_ms += kFrameIntervalMs;
6186 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6187 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 15:56:00 +01006188 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6189 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6190
Åsa Persson06defc42021-09-10 15:28:48 +02006191 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 15:56:00 +01006192 timestamp_ms += kFrameIntervalMs;
6193 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6194 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006195 // The ramp-up code involves the adaptation queue, give it time to execute.
6196 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006197 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006198 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006199
6200 // Frame should not be adapted.
6201 timestamp_ms += kFrameIntervalMs;
6202 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6203 WaitForEncodedFrame(kWidth, kHeight);
6204 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6205
6206 video_stream_encoder_->Stop();
6207}
6208
mflodmancc3d4422017-08-03 08:27:51 -07006209TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006210 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006211 webrtc::test::ScopedKeyValueConfig field_trials(
6212 field_trials_, "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006213 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006214 source.set_adaptation_enabled(true);
6215 video_stream_encoder_->SetSource(&source,
6216 DegradationPreference::MAINTAIN_FRAMERATE);
6217 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006218 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006219 fake_encoder_.SetQp(kQpHigh + 1);
6220 const int kWidth = 1280;
6221 const int kHeight = 720;
6222 const int64_t kFrameIntervalMs = 100;
6223 int64_t timestamp_ms = kFrameIntervalMs;
6224 for (size_t i = 1; i <= 100; i++) {
6225 timestamp_ms += kFrameIntervalMs;
6226 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6227 WaitForEncodedFrame(timestamp_ms);
6228 }
6229 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6230 // for the first time.
6231 // TODO(eshr): We should avoid these waits by using threads with simulated
6232 // time.
6233 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6234 2000 * 2.5 * 2);
6235 timestamp_ms += kFrameIntervalMs;
6236 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6237 WaitForEncodedFrame(timestamp_ms);
6238 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6239 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6240 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6241
6242 // Disable Quality scaling by turning off scaler on the encoder and
6243 // reconfiguring.
6244 fake_encoder_.SetQualityScaling(false);
6245 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6246 kMaxPayloadLength);
6247 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Markus Handell28c71802021-11-08 10:11:55 +01006248 AdvanceTime(TimeDelta::Zero());
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006249 // Since we turned off the quality scaler, the adaptations made by it are
6250 // removed.
6251 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6252 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6253
6254 video_stream_encoder_->Stop();
6255}
6256
6257TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006258 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6259 const int kTooSmallWidth = 10;
6260 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006261 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006262 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006263
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006264 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006265 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006266 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006267 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006268 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006269 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6270
6271 // Trigger adapt down, too small frame, expect no change.
6272 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006273 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006274 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006275 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006276 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6277 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6278
mflodmancc3d4422017-08-03 08:27:51 -07006279 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006280}
6281
mflodmancc3d4422017-08-03 08:27:51 -07006282TEST_F(VideoStreamEncoderTest,
6283 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006284 const int kTooSmallWidth = 10;
6285 const int kTooSmallHeight = 10;
6286 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006287 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006288 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006289
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006290 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006291 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006292 video_stream_encoder_->SetSource(&source,
6293 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006294 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006295 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6296 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6297
6298 // Trigger adapt down, expect limited framerate.
6299 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006300 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006301 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006302 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006303 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6304 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6305 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6306
6307 // Trigger adapt down, too small frame, expect no change.
6308 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006309 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006310 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006311 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006312 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6313 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6314 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6315
mflodmancc3d4422017-08-03 08:27:51 -07006316 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006317}
6318
mflodmancc3d4422017-08-03 08:27:51 -07006319TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006320 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006321 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006322 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006323 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006324 const int kFrameWidth = 1280;
6325 const int kFrameHeight = 720;
6326 video_source_.IncomingCapturedFrame(
6327 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006328 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006329 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006330}
6331
sprangb1ca0732017-02-01 08:38:12 -08006332// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006333TEST_F(VideoStreamEncoderTest,
6334 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006335 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006336 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006337
6338 const int kFrameWidth = 1280;
6339 const int kFrameHeight = 720;
6340 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006341 // requested by
6342 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006343 video_source_.set_adaptation_enabled(true);
6344
6345 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006346 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006347 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006348
6349 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006350 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006351 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006352 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006353 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006354
asaperssonfab67072017-04-04 05:51:49 -07006355 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006356 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006357 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006358 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006359 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006360
mflodmancc3d4422017-08-03 08:27:51 -07006361 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006362}
sprangfe627f32017-03-29 08:24:59 -07006363
mflodmancc3d4422017-08-03 08:27:51 -07006364TEST_F(VideoStreamEncoderTest,
6365 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006366 const int kFrameWidth = 1280;
6367 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006368
Henrik Boström381d1092020-05-12 18:49:07 +02006369 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006370 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006371 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006372 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006373 video_source_.set_adaptation_enabled(true);
6374
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006375 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006376
6377 video_source_.IncomingCapturedFrame(
6378 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006379 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006380
6381 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006382 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006383
6384 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006385 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006386 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006387 video_source_.IncomingCapturedFrame(
6388 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006389 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006390 }
6391
6392 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006393 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006394 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006395 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006396 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006397 video_source_.IncomingCapturedFrame(
6398 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006399 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006400 ++num_frames_dropped;
6401 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006402 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006403 }
6404 }
6405
sprang4847ae62017-06-27 07:06:52 -07006406 // Add some slack to account for frames dropped by the frame dropper.
6407 const int kErrorMargin = 1;
6408 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006409 kErrorMargin);
6410
6411 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006412 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006413 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006414 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006415 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006416 video_source_.IncomingCapturedFrame(
6417 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006418 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006419 ++num_frames_dropped;
6420 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006421 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006422 }
6423 }
sprang4847ae62017-06-27 07:06:52 -07006424 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006425 kErrorMargin);
6426
6427 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006428 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006429 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006430 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006431 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006432 video_source_.IncomingCapturedFrame(
6433 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006434 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006435 ++num_frames_dropped;
6436 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006437 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006438 }
6439 }
sprang4847ae62017-06-27 07:06:52 -07006440 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006441 kErrorMargin);
6442
6443 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006444 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006445 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006446 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006447 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006448 video_source_.IncomingCapturedFrame(
6449 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006450 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006451 ++num_frames_dropped;
6452 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006453 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006454 }
6455 }
6456 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6457
mflodmancc3d4422017-08-03 08:27:51 -07006458 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006459}
6460
mflodmancc3d4422017-08-03 08:27:51 -07006461TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006462 const int kFramerateFps = 5;
6463 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006464 const int kFrameWidth = 1280;
6465 const int kFrameHeight = 720;
6466
sprang4847ae62017-06-27 07:06:52 -07006467 // Reconfigure encoder with two temporal layers and screensharing, which will
6468 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006469 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006470
Henrik Boström381d1092020-05-12 18:49:07 +02006471 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006472 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006473 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006474 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006475 video_source_.set_adaptation_enabled(true);
6476
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006477 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006478
6479 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006480 rtc::VideoSinkWants last_wants;
6481 do {
6482 last_wants = video_source_.sink_wants();
6483
sprangc5d62e22017-04-02 23:53:04 -07006484 // Insert frames to get a new fps estimate...
6485 for (int j = 0; j < kFramerateFps; ++j) {
6486 video_source_.IncomingCapturedFrame(
6487 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006488 if (video_source_.last_sent_width()) {
6489 sink_.WaitForEncodedFrame(timestamp_ms);
6490 }
sprangc5d62e22017-04-02 23:53:04 -07006491 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006492 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006493 }
6494 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006495 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006496 } while (video_source_.sink_wants().max_framerate_fps <
6497 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006498
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006499 EXPECT_THAT(video_source_.sink_wants(),
6500 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006501
mflodmancc3d4422017-08-03 08:27:51 -07006502 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006503}
asaperssonf7e294d2017-06-13 23:25:22 -07006504
mflodmancc3d4422017-08-03 08:27:51 -07006505TEST_F(VideoStreamEncoderTest,
6506 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006507 const int kWidth = 1280;
6508 const int kHeight = 720;
6509 const int64_t kFrameIntervalMs = 150;
6510 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006511 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006512 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006513
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006514 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006515 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006516 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006517 video_stream_encoder_->SetSource(&source,
6518 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006519 timestamp_ms += kFrameIntervalMs;
6520 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006521 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006522 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006523 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6524 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6525 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6526
6527 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006528 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006529 timestamp_ms += kFrameIntervalMs;
6530 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006531 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006532 EXPECT_THAT(source.sink_wants(),
6533 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006534 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6535 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6536 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6537
6538 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006539 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006540 timestamp_ms += kFrameIntervalMs;
6541 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006542 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006543 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006544 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6545 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6546 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6547
6548 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006549 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006550 timestamp_ms += kFrameIntervalMs;
6551 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006552 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006553 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006554 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6555 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6556 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6557
6558 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006559 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006560 timestamp_ms += kFrameIntervalMs;
6561 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006562 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006563 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006564 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6565 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6566 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6567
6568 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006569 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006570 timestamp_ms += kFrameIntervalMs;
6571 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006572 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006573 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006574 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6575 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6576 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6577
6578 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006579 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006580 timestamp_ms += kFrameIntervalMs;
6581 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006582 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006583 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006584 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6585 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6586 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6587
6588 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006589 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006590 timestamp_ms += kFrameIntervalMs;
6591 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006592 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006593 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006594 rtc::VideoSinkWants last_wants = source.sink_wants();
6595 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6596 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6597 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6598
6599 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006600 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006601 timestamp_ms += kFrameIntervalMs;
6602 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006603 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006604 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006605 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6606 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6607 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6608
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006609 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006610 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006611 timestamp_ms += kFrameIntervalMs;
6612 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006613 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006614 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006615 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6616 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6617 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6618
6619 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006620 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006621 timestamp_ms += kFrameIntervalMs;
6622 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006623 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006624 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006625 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6626 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6627 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6628
6629 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006630 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006631 timestamp_ms += kFrameIntervalMs;
6632 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006633 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006634 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006635 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6636 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6637 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6638
6639 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006640 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006641 timestamp_ms += kFrameIntervalMs;
6642 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006643 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006644 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006645 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6646 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6647 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6648
6649 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006650 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006651 timestamp_ms += kFrameIntervalMs;
6652 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006653 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006654 EXPECT_THAT(source.sink_wants(), FpsMax());
6655 EXPECT_EQ(source.sink_wants().max_pixel_count,
6656 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006657 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6658 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6659 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6660
6661 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006662 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006663 timestamp_ms += kFrameIntervalMs;
6664 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006665 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006666 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006667 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6668 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6669 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6670
Åsa Persson30ab0152019-08-27 12:22:33 +02006671 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006672 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006673 timestamp_ms += kFrameIntervalMs;
6674 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006675 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006676 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006677 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006678 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6679 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6680 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6681
6682 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006683 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006684 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006685 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6686
mflodmancc3d4422017-08-03 08:27:51 -07006687 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006688}
6689
mflodmancc3d4422017-08-03 08:27:51 -07006690TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006691 const int kWidth = 1280;
6692 const int kHeight = 720;
6693 const int64_t kFrameIntervalMs = 150;
6694 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006695 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006696 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006697
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006698 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006699 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006700 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006701 video_stream_encoder_->SetSource(&source,
6702 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006703 timestamp_ms += kFrameIntervalMs;
6704 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006705 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006706 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006707 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6708 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6709 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6710 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6711 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6712 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6713
6714 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006715 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006716 timestamp_ms += kFrameIntervalMs;
6717 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006718 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006719 EXPECT_THAT(source.sink_wants(),
6720 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006721 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6722 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6723 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6724 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6725 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6726 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6727
6728 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006729 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006730 timestamp_ms += kFrameIntervalMs;
6731 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006732 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006733 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006734 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6735 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6736 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6737 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6738 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6739 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6740
6741 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006742 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006743 timestamp_ms += kFrameIntervalMs;
6744 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006745 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006746 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006747 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006748 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6749 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6750 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6751 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6752 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6753
Evan Shrubsole64469032020-06-11 10:45:29 +02006754 // Trigger cpu adapt up, expect no change since QP is most limited.
6755 {
6756 // Store current sink wants since we expect no change and if there is no
6757 // change then last_wants() is not updated.
6758 auto previous_sink_wants = source.sink_wants();
6759 video_stream_encoder_->TriggerCpuUnderuse();
6760 timestamp_ms += kFrameIntervalMs;
6761 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6762 WaitForEncodedFrame(timestamp_ms);
6763 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6764 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6765 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6766 }
6767
6768 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6769 video_stream_encoder_->TriggerQualityHigh();
6770 timestamp_ms += kFrameIntervalMs;
6771 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6772 WaitForEncodedFrame(timestamp_ms);
6773 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6774 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6775 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6776 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6777 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6778 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6779 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6780
6781 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6782 // expect increased resolution (960x540@30fps).
6783 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006784 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006785 timestamp_ms += kFrameIntervalMs;
6786 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006787 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006788 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006789 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6790 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6791 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6792 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6793 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006794 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006795
Evan Shrubsole64469032020-06-11 10:45:29 +02006796 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6797 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006798 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006799 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006800 timestamp_ms += kFrameIntervalMs;
6801 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006802 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006803 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006804 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006805 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6806 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6807 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6808 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6809 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006810 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006811
6812 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006813 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006814 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006815 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006816 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006817
mflodmancc3d4422017-08-03 08:27:51 -07006818 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006819}
6820
mflodmancc3d4422017-08-03 08:27:51 -07006821TEST_F(VideoStreamEncoderTest,
6822 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006823 const int kWidth = 640;
6824 const int kHeight = 360;
6825 const int kFpsLimit = 15;
6826 const int64_t kFrameIntervalMs = 150;
6827 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006828 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006829 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006830
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006831 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006832 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006833 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006834 video_stream_encoder_->SetSource(&source,
6835 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006836 timestamp_ms += kFrameIntervalMs;
6837 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006838 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006839 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006840 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6841 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6842 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6843 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6844 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6845 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6846
6847 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006848 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006849 timestamp_ms += kFrameIntervalMs;
6850 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006851 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006852 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006853 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6854 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6855 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6856 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6857 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6858 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6859
6860 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006861 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006862 timestamp_ms += kFrameIntervalMs;
6863 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006864 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006865 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006866 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006867 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006868 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6869 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6870 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6871 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6872
Evan Shrubsole64469032020-06-11 10:45:29 +02006873 // Trigger cpu adapt up, expect no change because quality is most limited.
6874 {
6875 auto previous_sink_wants = source.sink_wants();
6876 // Store current sink wants since we expect no change ind if there is no
6877 // change then last__wants() is not updated.
6878 video_stream_encoder_->TriggerCpuUnderuse();
6879 timestamp_ms += kFrameIntervalMs;
6880 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6881 WaitForEncodedFrame(timestamp_ms);
6882 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6883 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6884 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6885 }
6886
6887 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6888 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006889 timestamp_ms += kFrameIntervalMs;
6890 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006891 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006892 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006893 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6894 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6895 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006896 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6897 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6898 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006899
Evan Shrubsole64469032020-06-11 10:45:29 +02006900 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006901 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006902 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006903 timestamp_ms += kFrameIntervalMs;
6904 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006905 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006906 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006907 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6908 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6909 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6910 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6911 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006912 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006913
6914 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006915 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006916 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006917 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006918 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006919
mflodmancc3d4422017-08-03 08:27:51 -07006920 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006921}
6922
mflodmancc3d4422017-08-03 08:27:51 -07006923TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006924 const int kFrameWidth = 1920;
6925 const int kFrameHeight = 1080;
Ying Wangdb0d5862022-04-28 19:26:22 +02006926 // 2/3 of 1920.
6927 const int kAdaptedFrameWidth = 1280;
6928 // 2/3 of 1080.
6929 const int kAdaptedFrameHeight = 720;
ilnik6b826ef2017-06-16 06:53:48 -07006930 const int kFramerate = 24;
6931
Henrik Boström381d1092020-05-12 18:49:07 +02006932 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006933 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006934 // Trigger reconfigure encoder (without resetting the entire instance).
6935 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006936 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6937 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02006938 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 06:53:48 -07006939 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006940 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006941 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006942 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006943 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006944
6945 video_source_.set_adaptation_enabled(true);
6946
6947 video_source_.IncomingCapturedFrame(
6948 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006949 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006950
6951 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006952 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006953 video_source_.IncomingCapturedFrame(
6954 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006955 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006956
mflodmancc3d4422017-08-03 08:27:51 -07006957 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006958}
6959
mflodmancc3d4422017-08-03 08:27:51 -07006960TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006961 const int kFrameWidth = 1280;
6962 const int kFrameHeight = 720;
6963 const int kLowFps = 2;
6964 const int kHighFps = 30;
6965
Henrik Boström381d1092020-05-12 18:49:07 +02006966 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006967 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006968
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006969 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006970 max_framerate_ = kLowFps;
6971
6972 // Insert 2 seconds of 2fps video.
6973 for (int i = 0; i < kLowFps * 2; ++i) {
6974 video_source_.IncomingCapturedFrame(
6975 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6976 WaitForEncodedFrame(timestamp_ms);
6977 timestamp_ms += 1000 / kLowFps;
6978 }
6979
6980 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006981 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006982 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006983 video_source_.IncomingCapturedFrame(
6984 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6985 WaitForEncodedFrame(timestamp_ms);
6986 timestamp_ms += 1000 / kLowFps;
6987
6988 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6989
6990 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006991 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006992 const int kFrameIntervalMs = 1000 / kHighFps;
6993 max_framerate_ = kHighFps;
6994 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6995 video_source_.IncomingCapturedFrame(
6996 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6997 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6998 // be dropped if the encoder hans't been updated with the new higher target
6999 // framerate yet, causing it to overshoot the target bitrate and then
7000 // suffering the wrath of the media optimizer.
7001 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
7002 timestamp_ms += kFrameIntervalMs;
7003 }
7004
7005 // Don expect correct measurement just yet, but it should be higher than
7006 // before.
7007 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
7008
mflodmancc3d4422017-08-03 08:27:51 -07007009 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007010}
7011
mflodmancc3d4422017-08-03 08:27:51 -07007012TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07007013 const int kFrameWidth = 1280;
7014 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 15:09:05 +02007015 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01007016 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02007017 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07007018
Henrik Boström381d1092020-05-12 18:49:07 +02007019 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007020 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07007021 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07007022
7023 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007024 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07007025 video_source_.IncomingCapturedFrame(
7026 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7027 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02007028 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007029
7030 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02007031 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007032 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07007033
7034 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02007035 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007036 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07007037
Per Kjellanderdcef6412020-10-07 15:09:05 +02007038 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07007039 video_source_.IncomingCapturedFrame(
7040 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7041 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02007042 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007043
mflodmancc3d4422017-08-03 08:27:51 -07007044 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007045}
ilnik6b826ef2017-06-16 06:53:48 -07007046
Niels Möller4db138e2018-04-19 09:04:13 +02007047TEST_F(VideoStreamEncoderTest,
7048 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
7049 const int kFrameWidth = 1280;
7050 const int kFrameHeight = 720;
Markus Handell8e4197b2022-05-30 15:45:28 +02007051 const test::ScopedKeyValueConfig kFieldTrials;
7052 const CpuOveruseOptions default_options(kFieldTrials);
Henrik Boström381d1092020-05-12 18:49:07 +02007053 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007054 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007055 video_source_.IncomingCapturedFrame(
7056 CreateFrame(1, kFrameWidth, kFrameHeight));
7057 WaitForEncodedFrame(1);
7058 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7059 .low_encode_usage_threshold_percent,
7060 default_options.low_encode_usage_threshold_percent);
7061 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7062 .high_encode_usage_threshold_percent,
7063 default_options.high_encode_usage_threshold_percent);
7064 video_stream_encoder_->Stop();
7065}
7066
7067TEST_F(VideoStreamEncoderTest,
7068 HigherCpuAdaptationThresholdsForHardwareEncoder) {
7069 const int kFrameWidth = 1280;
7070 const int kFrameHeight = 720;
Markus Handell8e4197b2022-05-30 15:45:28 +02007071 const test::ScopedKeyValueConfig kFieldTrials;
7072 CpuOveruseOptions hardware_options(kFieldTrials);
Niels Möller4db138e2018-04-19 09:04:13 +02007073 hardware_options.low_encode_usage_threshold_percent = 150;
7074 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01007075 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02007076
Henrik Boström381d1092020-05-12 18:49:07 +02007077 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007078 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007079 video_source_.IncomingCapturedFrame(
7080 CreateFrame(1, kFrameWidth, kFrameHeight));
7081 WaitForEncodedFrame(1);
7082 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7083 .low_encode_usage_threshold_percent,
7084 hardware_options.low_encode_usage_threshold_percent);
7085 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7086 .high_encode_usage_threshold_percent,
7087 hardware_options.high_encode_usage_threshold_percent);
7088 video_stream_encoder_->Stop();
7089}
7090
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007091TEST_F(VideoStreamEncoderTest,
7092 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
7093 const int kFrameWidth = 1280;
7094 const int kFrameHeight = 720;
7095
Markus Handell8e4197b2022-05-30 15:45:28 +02007096 const test::ScopedKeyValueConfig kFieldTrials;
7097 const CpuOveruseOptions default_options(kFieldTrials);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007098 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007099 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007100 video_source_.IncomingCapturedFrame(
7101 CreateFrame(1, kFrameWidth, kFrameHeight));
7102 WaitForEncodedFrame(1);
7103 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7104 .low_encode_usage_threshold_percent,
7105 default_options.low_encode_usage_threshold_percent);
7106 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7107 .high_encode_usage_threshold_percent,
7108 default_options.high_encode_usage_threshold_percent);
7109
Markus Handell8e4197b2022-05-30 15:45:28 +02007110 CpuOveruseOptions hardware_options(kFieldTrials);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007111 hardware_options.low_encode_usage_threshold_percent = 150;
7112 hardware_options.high_encode_usage_threshold_percent = 200;
7113 fake_encoder_.SetIsHardwareAccelerated(true);
7114
7115 video_source_.IncomingCapturedFrame(
7116 CreateFrame(2, kFrameWidth, kFrameHeight));
7117 WaitForEncodedFrame(2);
7118
7119 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7120 .low_encode_usage_threshold_percent,
7121 hardware_options.low_encode_usage_threshold_percent);
7122 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7123 .high_encode_usage_threshold_percent,
7124 hardware_options.high_encode_usage_threshold_percent);
7125
7126 video_stream_encoder_->Stop();
7127}
7128
Niels Möller6bb5ab92019-01-11 11:11:10 +01007129TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7130 const int kFrameWidth = 320;
7131 const int kFrameHeight = 240;
7132 const int kFps = 30;
Asa Persson606d3cb2021-10-04 10:07:11 +02007133 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007134 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7135
Henrik Boström381d1092020-05-12 18:49:07 +02007136 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007137 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007138
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007139 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007140 max_framerate_ = kFps;
7141
7142 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7143 fake_encoder_.SimulateOvershoot(1.0);
7144 int num_dropped = 0;
7145 for (int i = 0; i < kNumFramesInRun; ++i) {
7146 video_source_.IncomingCapturedFrame(
7147 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7148 // Wait up to two frame durations for a frame to arrive.
7149 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7150 ++num_dropped;
7151 }
7152 timestamp_ms += 1000 / kFps;
7153 }
7154
Erik Språnga8d48ab2019-02-08 14:17:40 +01007155 // Framerate should be measured to be near the expected target rate.
7156 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7157
7158 // Frame drops should be within 5% of expected 0%.
7159 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007160
7161 // Make encoder produce frames at double the expected bitrate during 3 seconds
7162 // of video, verify number of drops. Rate needs to be slightly changed in
7163 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01007164 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02007165 const RateControlSettings trials =
7166 RateControlSettings::ParseFromFieldTrials();
7167 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01007168 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02007169 // frame dropping since the adjuter will try to just lower the target
7170 // bitrate rather than drop frames. If network headroom can be used, it
7171 // doesn't push back as hard so we don't need quite as much overshoot.
7172 // These numbers are unfortunately a bit magical but there's not trivial
7173 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01007174 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01007175 }
7176 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02007177 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007178 kTargetBitrate + DataRate::KilobitsPerSec(1),
7179 kTargetBitrate + DataRate::KilobitsPerSec(1),
7180 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007181 num_dropped = 0;
7182 for (int i = 0; i < kNumFramesInRun; ++i) {
7183 video_source_.IncomingCapturedFrame(
7184 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7185 // Wait up to two frame durations for a frame to arrive.
7186 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7187 ++num_dropped;
7188 }
7189 timestamp_ms += 1000 / kFps;
7190 }
7191
Henrik Boström381d1092020-05-12 18:49:07 +02007192 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007193 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007194
7195 // Target framerate should be still be near the expected target, despite
7196 // the frame drops.
7197 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7198
7199 // Frame drops should be within 5% of expected 50%.
7200 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007201
7202 video_stream_encoder_->Stop();
7203}
7204
7205TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7206 const int kFrameWidth = 320;
7207 const int kFrameHeight = 240;
7208 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 10:07:11 +02007209 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007210
7211 ASSERT_GT(max_framerate_, kActualInputFps);
7212
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007213 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007214 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007215 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007216 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007217
7218 // Insert 3 seconds of video, with an input fps lower than configured max.
7219 for (int i = 0; i < kActualInputFps * 3; ++i) {
7220 video_source_.IncomingCapturedFrame(
7221 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7222 // Wait up to two frame durations for a frame to arrive.
7223 WaitForEncodedFrame(timestamp_ms);
7224 timestamp_ms += 1000 / kActualInputFps;
7225 }
7226
7227 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7228
7229 video_stream_encoder_->Stop();
7230}
7231
Markus Handell9a478b52021-11-18 16:07:01 +01007232TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007233 VideoFrame::UpdateRect rect;
Markus Handell9a478b52021-11-18 16:07:01 +01007234 test::FrameForwarder source;
7235 video_stream_encoder_->SetSource(&source,
7236 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02007237 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007238 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007239
Markus Handell9a478b52021-11-18 16:07:01 +01007240 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(1, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007241 WaitForEncodedFrame(1);
7242 // On the very first frame full update should be forced.
7243 rect = fake_encoder_.GetLastUpdateRect();
7244 EXPECT_EQ(rect.offset_x, 0);
7245 EXPECT_EQ(rect.offset_y, 0);
7246 EXPECT_EQ(rect.height, codec_height_);
7247 EXPECT_EQ(rect.width, codec_width_);
Markus Handell9a478b52021-11-18 16:07:01 +01007248 // Frame with NTP timestamp 2 will be dropped due to outstanding frames
7249 // scheduled for processing during encoder queue processing of frame 2.
7250 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(2, nullptr, 1));
7251 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(3, nullptr, 10));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007252 WaitForEncodedFrame(3);
7253 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7254 rect = fake_encoder_.GetLastUpdateRect();
7255 EXPECT_EQ(rect.offset_x, 1);
7256 EXPECT_EQ(rect.offset_y, 0);
7257 EXPECT_EQ(rect.width, 10);
7258 EXPECT_EQ(rect.height, 1);
7259
Markus Handell9a478b52021-11-18 16:07:01 +01007260 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(4, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007261 WaitForEncodedFrame(4);
7262 // Previous frame was encoded, so no accumulation should happen.
7263 rect = fake_encoder_.GetLastUpdateRect();
7264 EXPECT_EQ(rect.offset_x, 0);
7265 EXPECT_EQ(rect.offset_y, 0);
7266 EXPECT_EQ(rect.width, 1);
7267 EXPECT_EQ(rect.height, 1);
7268
7269 video_stream_encoder_->Stop();
7270}
7271
Erik Språngd7329ca2019-02-21 21:19:53 +01007272TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007273 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007274 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007275
7276 // First frame is always keyframe.
7277 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7278 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007279 EXPECT_THAT(
7280 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007281 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007282
7283 // Insert delta frame.
7284 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7285 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007286 EXPECT_THAT(
7287 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007288 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007289
7290 // Request next frame be a key-frame.
7291 video_stream_encoder_->SendKeyFrame();
7292 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7293 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007294 EXPECT_THAT(
7295 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007296 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007297
7298 video_stream_encoder_->Stop();
7299}
7300
7301TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7302 // Setup simulcast with three streams.
7303 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007304 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007305 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7306 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007307 // Wait for all three layers before triggering event.
7308 sink_.SetNumExpectedLayers(3);
7309
7310 // First frame is always keyframe.
7311 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7312 WaitForEncodedFrame(1);
7313 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007314 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7315 VideoFrameType::kVideoFrameKey,
7316 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007317
7318 // Insert delta frame.
7319 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7320 WaitForEncodedFrame(2);
7321 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007322 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7323 VideoFrameType::kVideoFrameDelta,
7324 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007325
7326 // Request next frame be a key-frame.
7327 // Only first stream is configured to produce key-frame.
7328 video_stream_encoder_->SendKeyFrame();
7329 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7330 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007331
7332 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7333 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007334 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007335 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007336 VideoFrameType::kVideoFrameKey,
7337 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007338
7339 video_stream_encoder_->Stop();
7340}
7341
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007342TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007343 // SPS contains VUI with restrictions on the maximum number of reordered
7344 // pictures, there is no need to rewrite the bitstream to enable faster
7345 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007346 ResetEncoder("H264", 1, 1, 1, false);
7347
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007348 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007349 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007350 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007351
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007352 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 10:07:11 +02007353 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007354
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007355 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7356 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007357
7358 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007359 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007360
7361 video_stream_encoder_->Stop();
7362}
7363
7364TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007365 // SPS does not contain VUI, the bitstream is will be rewritten with added
7366 // VUI with restrictions on the maximum number of reordered pictures to
7367 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007368 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7369 0x00, 0x00, 0x03, 0x03, 0xF4,
7370 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007371 ResetEncoder("H264", 1, 1, 1, false);
7372
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007373 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007374 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007375 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007376
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007377 fake_encoder_.SetEncodedImageData(
7378 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007379
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007380 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7381 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007382
7383 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007384 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007385
7386 video_stream_encoder_->Stop();
7387}
7388
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007389TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7390 const int kFrameWidth = 1280;
7391 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 10:07:11 +02007392 const DataRate kTargetBitrate =
7393 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007394
Henrik Boström381d1092020-05-12 18:49:07 +02007395 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007396 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007397 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7398
7399 // Insert a first video frame. It should be dropped because of downscale in
7400 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007401 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007402 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7403 frame.set_rotation(kVideoRotation_270);
7404 video_source_.IncomingCapturedFrame(frame);
7405
7406 ExpectDroppedFrame();
7407
7408 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007409 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007410 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7411 frame.set_rotation(kVideoRotation_90);
7412 video_source_.IncomingCapturedFrame(frame);
7413
7414 WaitForEncodedFrame(timestamp_ms);
7415 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7416
7417 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007418 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007419 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7420 frame.set_rotation(kVideoRotation_180);
7421 video_source_.IncomingCapturedFrame(frame);
7422
7423 WaitForEncodedFrame(timestamp_ms);
7424 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7425
7426 video_stream_encoder_->Stop();
7427}
7428
Erik Språng5056af02019-09-02 15:53:11 +02007429TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7430 const int kFrameWidth = 320;
7431 const int kFrameHeight = 180;
7432
7433 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007434 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007435 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7436 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7437 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007438 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007439 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007440 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007441
7442 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007443 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007444 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7445 frame.set_rotation(kVideoRotation_270);
7446 video_source_.IncomingCapturedFrame(frame);
7447 WaitForEncodedFrame(timestamp_ms);
7448
7449 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 10:07:11 +02007450 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007451 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7452 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007453 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007454 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007455 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007456 /*link_allocation=*/target_rate,
7457 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007458 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007459 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007460 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7461
7462 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7463 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7464 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007465 DataRate allocation_sum =
7466 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007467 EXPECT_EQ(min_rate, allocation_sum);
7468 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7469
7470 video_stream_encoder_->Stop();
7471}
7472
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007473TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007474 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007475 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007476 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007477 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007478 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7479 WaitForEncodedFrame(1);
7480
7481 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7482 ASSERT_TRUE(prev_rate_settings.has_value());
7483 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7484 kDefaultFramerate);
7485
7486 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7487 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7488 timestamp_ms += 1000 / kDefaultFramerate;
7489 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7490 WaitForEncodedFrame(timestamp_ms);
7491 }
7492 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7493 kDefaultFramerate);
7494 // Capture larger frame to trigger a reconfigure.
7495 codec_height_ *= 2;
7496 codec_width_ *= 2;
7497 timestamp_ms += 1000 / kDefaultFramerate;
7498 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7499 WaitForEncodedFrame(timestamp_ms);
7500
7501 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7502 auto current_rate_settings =
7503 fake_encoder_.GetAndResetLastRateControlSettings();
7504 // Ensure we have actually reconfigured twice
7505 // The rate settings should have been set again even though
7506 // they haven't changed.
7507 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007508 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007509
7510 video_stream_encoder_->Stop();
7511}
7512
philipeld9cc8c02019-09-16 14:53:40 +02007513struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007514 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007515 MOCK_METHOD(void,
7516 RequestEncoderSwitch,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007517 (const webrtc::SdpVideoFormat& format,
7518 bool allow_default_fallback),
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007519 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007520};
7521
philipel9b058032020-02-10 11:30:00 +01007522TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7523 constexpr int kDontCare = 100;
7524 StrictMock<MockEncoderSelector> encoder_selector;
7525 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7526 &fake_encoder_, &encoder_selector);
7527 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7528
7529 // Reset encoder for new configuration to take effect.
7530 ConfigureEncoder(video_encoder_config_.Copy());
7531
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007532 EXPECT_CALL(encoder_selector, OnCurrentEncoder);
philipel9b058032020-02-10 11:30:00 +01007533
7534 video_source_.IncomingCapturedFrame(
7535 CreateFrame(kDontCare, kDontCare, kDontCare));
Markus Handell28c71802021-11-08 10:11:55 +01007536 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007537 video_stream_encoder_->Stop();
7538
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007539 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007540 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007541 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7542 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007543 video_stream_encoder_.reset();
7544}
7545
7546TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7547 constexpr int kDontCare = 100;
7548
7549 NiceMock<MockEncoderSelector> encoder_selector;
7550 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7551 video_send_config_.encoder_settings.encoder_switch_request_callback =
7552 &switch_callback;
7553 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7554 &fake_encoder_, &encoder_selector);
7555 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7556
7557 // Reset encoder for new configuration to take effect.
7558 ConfigureEncoder(video_encoder_config_.Copy());
7559
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007560 ON_CALL(encoder_selector, OnAvailableBitrate)
philipel9b058032020-02-10 11:30:00 +01007561 .WillByDefault(Return(SdpVideoFormat("AV1")));
7562 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007563 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7564 /*allow_default_fallback=*/false));
philipel9b058032020-02-10 11:30:00 +01007565
Henrik Boström381d1092020-05-12 18:49:07 +02007566 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007567 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7568 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7569 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007570 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007571 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007572 /*cwnd_reduce_ratio=*/0);
Markus Handell28c71802021-11-08 10:11:55 +01007573 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007574
7575 video_stream_encoder_->Stop();
7576}
7577
philipel6daa3042022-04-11 10:48:28 +02007578TEST_F(VideoStreamEncoderTest, EncoderSelectorResolutionSwitch) {
7579 NiceMock<MockEncoderSelector> encoder_selector;
7580 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7581 video_send_config_.encoder_settings.encoder_switch_request_callback =
7582 &switch_callback;
7583 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7584 &fake_encoder_, &encoder_selector);
7585 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7586
7587 // Reset encoder for new configuration to take effect.
7588 ConfigureEncoder(video_encoder_config_.Copy());
7589
7590 EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(640, 480)))
7591 .WillOnce(Return(absl::nullopt));
7592 EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(320, 240)))
7593 .WillOnce(Return(SdpVideoFormat("AV1")));
7594 EXPECT_CALL(switch_callback,
7595 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7596 /*allow_default_fallback=*/false));
7597
7598 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7599 /*target_bitrate=*/DataRate::KilobitsPerSec(800),
7600 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(1000),
7601 /*link_allocation=*/DataRate::KilobitsPerSec(1000),
7602 /*fraction_lost=*/0,
7603 /*round_trip_time_ms=*/0,
7604 /*cwnd_reduce_ratio=*/0);
7605
7606 video_source_.IncomingCapturedFrame(CreateFrame(1, 640, 480));
7607 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 480));
7608 video_source_.IncomingCapturedFrame(CreateFrame(3, 320, 240));
7609
7610 AdvanceTime(TimeDelta::Zero());
7611
7612 video_stream_encoder_->Stop();
7613}
7614
philipel9b058032020-02-10 11:30:00 +01007615TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7616 constexpr int kSufficientBitrateToNotDrop = 1000;
7617 constexpr int kDontCare = 100;
7618
7619 NiceMock<MockVideoEncoder> video_encoder;
7620 NiceMock<MockEncoderSelector> encoder_selector;
7621 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7622 video_send_config_.encoder_settings.encoder_switch_request_callback =
7623 &switch_callback;
7624 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7625 &video_encoder, &encoder_selector);
7626 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7627
7628 // Reset encoder for new configuration to take effect.
7629 ConfigureEncoder(video_encoder_config_.Copy());
7630
7631 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7632 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7633 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007634 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007635 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7636 /*stable_target_bitrate=*/
7637 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7638 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007639 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007640 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007641 /*cwnd_reduce_ratio=*/0);
7642
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007643 ON_CALL(video_encoder, Encode)
philipel9b058032020-02-10 11:30:00 +01007644 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007645 ON_CALL(encoder_selector, OnEncoderBroken)
philipel9b058032020-02-10 11:30:00 +01007646 .WillByDefault(Return(SdpVideoFormat("AV2")));
7647
7648 rtc::Event encode_attempted;
7649 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007650 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7651 /*allow_default_fallback=*/true))
7652 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
philipel9b058032020-02-10 11:30:00 +01007653
7654 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7655 encode_attempted.Wait(3000);
7656
Markus Handell28c71802021-11-08 10:11:55 +01007657 AdvanceTime(TimeDelta::Zero());
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007658
philipel9b058032020-02-10 11:30:00 +01007659 video_stream_encoder_->Stop();
7660
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007661 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7662 // to it's factory, so in order for the encoder instance in the
7663 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7664 // reset the `video_stream_encoder_` here.
7665 video_stream_encoder_.reset();
7666}
7667
7668TEST_F(VideoStreamEncoderTest, SwitchEncoderOnInitFailureWithEncoderSelector) {
7669 NiceMock<MockVideoEncoder> video_encoder;
7670 NiceMock<MockEncoderSelector> encoder_selector;
7671 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7672 video_send_config_.encoder_settings.encoder_switch_request_callback =
7673 &switch_callback;
7674 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7675 &video_encoder, &encoder_selector);
7676 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7677
7678 // Reset encoder for new configuration to take effect.
7679 ConfigureEncoder(video_encoder_config_.Copy());
7680
7681 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7682 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7683 /*round_trip_time_ms=*/0,
7684 /*cwnd_reduce_ratio=*/0);
7685 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7686
7687 ON_CALL(video_encoder, InitEncode(_, _))
7688 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7689 ON_CALL(encoder_selector, OnEncoderBroken)
7690 .WillByDefault(Return(SdpVideoFormat("AV2")));
7691
7692 rtc::Event encode_attempted;
7693 EXPECT_CALL(switch_callback,
7694 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7695 /*allow_default_fallback=*/true))
7696 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7697
7698 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7699 encode_attempted.Wait(3000);
7700
7701 AdvanceTime(TimeDelta::Zero());
7702
7703 video_stream_encoder_->Stop();
7704
7705 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7706 // to it's factory, so in order for the encoder instance in the
7707 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7708 // reset the `video_stream_encoder_` here.
7709 video_stream_encoder_.reset();
7710}
7711
7712TEST_F(VideoStreamEncoderTest,
7713 SwitchEncoderOnInitFailureWithoutEncoderSelector) {
7714 NiceMock<MockVideoEncoder> video_encoder;
7715 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7716 video_send_config_.encoder_settings.encoder_switch_request_callback =
7717 &switch_callback;
7718 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7719 &video_encoder, /*encoder_selector=*/nullptr);
7720 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7721
7722 // Reset encoder for new configuration to take effect.
7723 ConfigureEncoder(video_encoder_config_.Copy());
7724
7725 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7726 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7727 /*round_trip_time_ms=*/0,
7728 /*cwnd_reduce_ratio=*/0);
7729 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7730
7731 ON_CALL(video_encoder, InitEncode(_, _))
7732 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7733
7734 rtc::Event encode_attempted;
7735 EXPECT_CALL(switch_callback,
7736 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "VP8"),
7737 /*allow_default_fallback=*/true))
7738 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7739
7740 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7741 encode_attempted.Wait(3000);
7742
7743 AdvanceTime(TimeDelta::Zero());
7744
7745 video_stream_encoder_->Stop();
7746
7747 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007748 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007749 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7750 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007751 video_stream_encoder_.reset();
7752}
7753
Byoungchan Lee13fe3672022-04-06 10:44:42 +09007754TEST_F(VideoStreamEncoderTest, NullEncoderReturnSwitch) {
7755 // As a variant of EncoderSelectorBrokenEncoderSwitch, when a null
7756 // VideoEncoder is passed in encoder_factory, it checks whether
7757 // Codec Switch occurs without a crash.
7758 constexpr int kSufficientBitrateToNotDrop = 1000;
7759 constexpr int kDontCare = 100;
7760
7761 NiceMock<MockEncoderSelector> encoder_selector;
7762 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7763 video_send_config_.encoder_settings.encoder_switch_request_callback =
7764 &switch_callback;
7765 auto encoder_factory =
7766 std::make_unique<test::VideoEncoderNullableProxyFactory>(
7767 /*encoder=*/nullptr, &encoder_selector);
7768 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7769
7770 // Reset encoder for new configuration to take effect.
7771 ConfigureEncoder(video_encoder_config_.Copy());
7772 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7773 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7774 // not fail.
7775 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7776 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7777 /*stable_target_bitrate=*/
7778 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7779 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7780 /*fraction_lost=*/0,
7781 /*round_trip_time_ms=*/0,
7782 /*cwnd_reduce_ratio=*/0);
7783 ON_CALL(encoder_selector, OnEncoderBroken)
7784 .WillByDefault(Return(SdpVideoFormat("AV2")));
7785 rtc::Event encode_attempted;
7786 EXPECT_CALL(switch_callback,
7787 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7788 /*allow_default_fallback=*/_))
7789 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7790
7791 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7792 encode_attempted.Wait(3000);
7793
7794 AdvanceTime(TimeDelta::Zero());
7795
7796 video_stream_encoder_->Stop();
7797
7798 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7799 // to it's factory, so in order for the encoder instance in the
7800 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7801 // reset the `video_stream_encoder_` here.
7802 video_stream_encoder_.reset();
7803}
7804
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007805TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007806 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007807 const int kFrameWidth = 320;
7808 const int kFrameHeight = 180;
7809
7810 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007811 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007812 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007813 /*target_bitrate=*/rate,
7814 /*stable_target_bitrate=*/rate,
7815 /*link_allocation=*/rate,
7816 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007817 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007818 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007819
7820 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007821 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007822 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7823 frame.set_rotation(kVideoRotation_270);
7824 video_source_.IncomingCapturedFrame(frame);
7825 WaitForEncodedFrame(timestamp_ms);
7826 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7827
7828 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007829 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007830 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007831 /*target_bitrate=*/new_stable_rate,
7832 /*stable_target_bitrate=*/new_stable_rate,
7833 /*link_allocation=*/rate,
7834 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007835 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007836 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007837 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7838 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7839 video_stream_encoder_->Stop();
7840}
7841
7842TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007843 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007844 const int kFrameWidth = 320;
7845 const int kFrameHeight = 180;
7846
7847 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007848 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007849 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007850 /*target_bitrate=*/rate,
7851 /*stable_target_bitrate=*/rate,
7852 /*link_allocation=*/rate,
7853 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007854 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007855 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007856
7857 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007858 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007859 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7860 frame.set_rotation(kVideoRotation_270);
7861 video_source_.IncomingCapturedFrame(frame);
7862 WaitForEncodedFrame(timestamp_ms);
7863 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7864
7865 // Set a higher target rate without changing the link_allocation. Should not
7866 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007867 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007868 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007869 /*target_bitrate=*/rate,
7870 /*stable_target_bitrate=*/new_stable_rate,
7871 /*link_allocation=*/rate,
7872 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007873 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007874 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007875 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7876 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7877 video_stream_encoder_->Stop();
7878}
7879
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007880TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01007881 test::ScopedKeyValueConfig field_trials(
7882 field_trials_,
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007883 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7884 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7885 const int kFramerateFps = 30;
7886 const int kWidth = 1920;
7887 const int kHeight = 1080;
7888 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7889 // Works on screenshare mode.
7890 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7891 // We rely on the automatic resolution adaptation, but we handle framerate
7892 // adaptation manually by mocking the stats proxy.
7893 video_source_.set_adaptation_enabled(true);
7894
7895 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007896 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007897 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007898 video_stream_encoder_->SetSource(&video_source_,
7899 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007900 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007901
7902 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7903 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7904
7905 // Pass enough frames with the full update to trigger animation detection.
7906 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007907 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007908 frame.set_ntp_time_ms(timestamp_ms);
7909 frame.set_timestamp_us(timestamp_ms * 1000);
7910 video_source_.IncomingCapturedFrame(frame);
7911 WaitForEncodedFrame(timestamp_ms);
7912 }
7913
7914 // Resolution should be limited.
7915 rtc::VideoSinkWants expected;
7916 expected.max_framerate_fps = kFramerateFps;
7917 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007918 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007919
7920 // Pass one frame with no known update.
7921 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007922 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007923 frame.set_ntp_time_ms(timestamp_ms);
7924 frame.set_timestamp_us(timestamp_ms * 1000);
7925 frame.clear_update_rect();
7926
7927 video_source_.IncomingCapturedFrame(frame);
7928 WaitForEncodedFrame(timestamp_ms);
7929
7930 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007931 EXPECT_THAT(video_source_.sink_wants(),
7932 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007933
7934 video_stream_encoder_->Stop();
7935}
7936
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007937TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7938 const int kWidth = 720; // 540p adapted down.
7939 const int kHeight = 405;
7940 const int kNumFrames = 3;
7941 // Works on screenshare mode.
7942 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7943 /*num_spatial_layers=*/2, /*screenshare=*/true);
7944
7945 video_source_.set_adaptation_enabled(true);
7946
7947 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007948 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007949
7950 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7951
7952 // Pass enough frames with the full update to trigger animation detection.
7953 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007954 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007955 frame.set_ntp_time_ms(timestamp_ms);
7956 frame.set_timestamp_us(timestamp_ms * 1000);
7957 video_source_.IncomingCapturedFrame(frame);
7958 WaitForEncodedFrame(timestamp_ms);
7959 }
7960
7961 video_stream_encoder_->Stop();
7962}
7963
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007964TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7965 const float downscale_factors[] = {4.0, 2.0, 1.0};
7966 const int number_layers =
7967 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7968 VideoEncoderConfig config;
7969 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7970 for (int i = 0; i < number_layers; ++i) {
7971 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7972 config.simulcast_layers[i].active = true;
7973 }
7974 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007975 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007976 "VP8", /*max qp*/ 56, /*screencast*/ false,
7977 /*screenshare enabled*/ false);
7978 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007979 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7980 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007981
7982 // First initialization.
7983 // Encoder should be initialized. Next frame should be key frame.
7984 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7985 sink_.SetNumExpectedLayers(number_layers);
7986 int64_t timestamp_ms = kFrameIntervalMs;
7987 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7988 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007989 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007990 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7991 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7992 VideoFrameType::kVideoFrameKey,
7993 VideoFrameType::kVideoFrameKey}));
7994
7995 // Disable top layer.
7996 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7997 config.simulcast_layers[number_layers - 1].active = false;
7998 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7999 sink_.SetNumExpectedLayers(number_layers - 1);
8000 timestamp_ms += kFrameIntervalMs;
8001 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8002 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008003 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008004 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8005 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
8006 VideoFrameType::kVideoFrameDelta,
8007 VideoFrameType::kVideoFrameDelta}));
8008
8009 // Re-enable top layer.
8010 // Encoder should be re-initialized. Next frame should be key frame.
8011 config.simulcast_layers[number_layers - 1].active = true;
8012 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8013 sink_.SetNumExpectedLayers(number_layers);
8014 timestamp_ms += kFrameIntervalMs;
8015 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8016 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008017 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008018 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8019 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
8020 VideoFrameType::kVideoFrameKey,
8021 VideoFrameType::kVideoFrameKey}));
8022
8023 // Top layer max rate change.
8024 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
8025 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
8026 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8027 sink_.SetNumExpectedLayers(number_layers);
8028 timestamp_ms += kFrameIntervalMs;
8029 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8030 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008031 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008032 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8033 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
8034 VideoFrameType::kVideoFrameDelta,
8035 VideoFrameType::kVideoFrameDelta}));
8036
8037 // Top layer resolution change.
8038 // Encoder should be re-initialized. Next frame should be key frame.
8039 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
8040 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8041 sink_.SetNumExpectedLayers(number_layers);
8042 timestamp_ms += kFrameIntervalMs;
8043 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8044 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008045 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008046 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8047 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
8048 VideoFrameType::kVideoFrameKey,
8049 VideoFrameType::kVideoFrameKey}));
8050 video_stream_encoder_->Stop();
8051}
8052
Henrik Boström1124ed12021-02-25 10:30:39 +01008053TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
8054 const int kFrameWidth = 1280;
8055 const int kFrameHeight = 720;
8056
8057 SetUp();
8058 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008059 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01008060
8061 // Capturing a frame should reconfigure the encoder and expose the encoder
8062 // resolution, which is the same as the input frame.
8063 int64_t timestamp_ms = kFrameIntervalMs;
8064 video_source_.IncomingCapturedFrame(
8065 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8066 WaitForEncodedFrame(timestamp_ms);
8067 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8068 EXPECT_THAT(video_source_.sink_wants().resolutions,
8069 ::testing::ElementsAreArray(
8070 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
8071
8072 video_stream_encoder_->Stop();
8073}
8074
8075TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
8076 // Pick downscale factors such that we never encode at full resolution - this
8077 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 16:22:11 +02008078 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 10:30:39 +01008079 // encoder should not ask for the frame resolution. This allows video frames
8080 // to have the appearence of one resolution but optimize its internal buffers
8081 // for what is actually encoded.
8082 const size_t kNumSimulcastLayers = 3u;
8083 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
8084 const int kFrameWidth = 1280;
8085 const int kFrameHeight = 720;
8086 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8087 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8088 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8089 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8090 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8091 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8092
8093 VideoEncoderConfig config;
8094 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
8095 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
8096 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
8097 config.simulcast_layers[i].active = true;
8098 }
8099 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02008100 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01008101 "VP8", /*max qp*/ 56, /*screencast*/ false,
8102 /*screenshare enabled*/ false);
8103 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008104 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8105 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01008106
8107 // Capture a frame with all layers active.
8108 int64_t timestamp_ms = kFrameIntervalMs;
8109 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
8110 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8111 video_source_.IncomingCapturedFrame(
8112 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8113 WaitForEncodedFrame(timestamp_ms);
8114 // Expect encoded resolutions to match the expected simulcast layers.
8115 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8116 EXPECT_THAT(
8117 video_source_.sink_wants().resolutions,
8118 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
8119
8120 // Capture a frame with one of the layers inactive.
8121 timestamp_ms += kFrameIntervalMs;
8122 config.simulcast_layers[2].active = false;
8123 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
8124 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8125 video_source_.IncomingCapturedFrame(
8126 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8127 WaitForEncodedFrame(timestamp_ms);
8128
8129 // Expect encoded resolutions to match the expected simulcast layers.
8130 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8131 EXPECT_THAT(video_source_.sink_wants().resolutions,
8132 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
8133
8134 // Capture a frame with all but one layer turned off.
8135 timestamp_ms += kFrameIntervalMs;
8136 config.simulcast_layers[1].active = false;
8137 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
8138 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8139 video_source_.IncomingCapturedFrame(
8140 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8141 WaitForEncodedFrame(timestamp_ms);
8142
8143 // Expect encoded resolutions to match the expected simulcast layers.
8144 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8145 EXPECT_THAT(video_source_.sink_wants().resolutions,
8146 ::testing::ElementsAreArray({kLayer0Size}));
8147
8148 video_stream_encoder_->Stop();
8149}
8150
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008151TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008152 ResetEncoder("VP8", 1, 1, 1, false);
8153
Niels Möller8b692902021-06-14 12:04:57 +02008154 // Force encoder reconfig.
8155 video_source_.IncomingCapturedFrame(
8156 CreateFrame(1, codec_width_, codec_height_));
8157 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8158
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008159 // Set QP on encoded frame and pass the frame to encode complete callback.
8160 // Since QP is present QP parsing won't be triggered and the original value
8161 // should be kept.
8162 EncodedImage encoded_image;
8163 encoded_image.qp_ = 123;
8164 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8165 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8166 CodecSpecificInfo codec_info;
8167 codec_info.codecType = kVideoCodecVP8;
8168 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8169 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8170 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
8171 video_stream_encoder_->Stop();
8172}
8173
8174TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008175 ResetEncoder("VP8", 1, 1, 1, false);
8176
Niels Möller8b692902021-06-14 12:04:57 +02008177 // Force encoder reconfig.
8178 video_source_.IncomingCapturedFrame(
8179 CreateFrame(1, codec_width_, codec_height_));
8180 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8181
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008182 // Pass an encoded frame without QP to encode complete callback. QP should be
8183 // parsed and set.
8184 EncodedImage encoded_image;
8185 encoded_image.qp_ = -1;
8186 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8187 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8188 CodecSpecificInfo codec_info;
8189 codec_info.codecType = kVideoCodecVP8;
8190 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8191 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8192 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
8193 video_stream_encoder_->Stop();
8194}
8195
8196TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01008197 webrtc::test::ScopedKeyValueConfig field_trials(
8198 field_trials_, "WebRTC-QpParsingKillSwitch/Enabled/");
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008199
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008200 ResetEncoder("VP8", 1, 1, 1, false);
8201
Niels Möller8b692902021-06-14 12:04:57 +02008202 // Force encoder reconfig.
8203 video_source_.IncomingCapturedFrame(
8204 CreateFrame(1, codec_width_, codec_height_));
8205 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8206
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008207 EncodedImage encoded_image;
8208 encoded_image.qp_ = -1;
8209 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8210 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8211 CodecSpecificInfo codec_info;
8212 codec_info.codecType = kVideoCodecVP8;
8213 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8214 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8215 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
8216 video_stream_encoder_->Stop();
8217}
8218
Sergey Silkind19e3b92021-03-16 10:05:30 +00008219TEST_F(VideoStreamEncoderTest,
8220 QualityScalingNotAllowed_QualityScalingDisabled) {
8221 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8222
8223 // Disable scaling settings in encoder info.
8224 fake_encoder_.SetQualityScaling(false);
8225 // Disable quality scaling in encoder config.
8226 video_encoder_config.is_quality_scaling_allowed = false;
8227 ConfigureEncoder(std::move(video_encoder_config));
8228
8229 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008230 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008231
8232 test::FrameForwarder source;
8233 video_stream_encoder_->SetSource(
8234 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8235 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8236 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8237
8238 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8239 WaitForEncodedFrame(1);
8240 video_stream_encoder_->TriggerQualityLow();
8241 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8242
8243 video_stream_encoder_->Stop();
8244}
8245
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008246TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
8247 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8248
8249 // Disable scaling settings in encoder info.
8250 fake_encoder_.SetQualityScaling(false);
8251 // Set QP trusted in encoder info.
8252 fake_encoder_.SetIsQpTrusted(true);
8253 // Enable quality scaling in encoder config.
8254 video_encoder_config.is_quality_scaling_allowed = false;
8255 ConfigureEncoder(std::move(video_encoder_config));
8256
8257 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008258 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008259
8260 test::FrameForwarder source;
8261 video_stream_encoder_->SetSource(
8262 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8263 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8264 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8265
8266 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8267 WaitForEncodedFrame(1);
8268 video_stream_encoder_->TriggerQualityLow();
8269 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8270
8271 video_stream_encoder_->Stop();
8272}
8273
Shuhai Pengf2707702021-09-29 17:19:44 +08008274TEST_F(VideoStreamEncoderTest,
8275 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8276 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8277
8278 // Disable scaling settings in encoder info.
8279 fake_encoder_.SetQualityScaling(false);
8280 // Set QP trusted in encoder info.
8281 fake_encoder_.SetIsQpTrusted(true);
8282 // Enable quality scaling in encoder config.
8283 video_encoder_config.is_quality_scaling_allowed = false;
8284 ConfigureEncoder(std::move(video_encoder_config));
8285
8286 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008287 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008288
8289 test::FrameForwarder source;
8290 video_stream_encoder_->SetSource(
8291 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8292 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8293 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8294
8295 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8296 WaitForEncodedFrame(1);
8297 video_stream_encoder_->TriggerQualityLow();
8298 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8299
8300 video_stream_encoder_->Stop();
8301}
8302
8303TEST_F(VideoStreamEncoderTest,
8304 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8305 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8306
8307 // Disable scaling settings in encoder info.
8308 fake_encoder_.SetQualityScaling(false);
8309 // Set QP trusted in encoder info.
8310 fake_encoder_.SetIsQpTrusted(false);
8311 // Enable quality scaling in encoder config.
8312 video_encoder_config.is_quality_scaling_allowed = false;
8313 ConfigureEncoder(std::move(video_encoder_config));
8314
8315 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008316 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008317
8318 test::FrameForwarder source;
8319 video_stream_encoder_->SetSource(
8320 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8321 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8322 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8323
8324 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8325 WaitForEncodedFrame(1);
8326 video_stream_encoder_->TriggerQualityLow();
8327 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8328
8329 video_stream_encoder_->Stop();
8330}
8331
8332TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8333 // Set QP trusted in encoder info.
8334 fake_encoder_.SetIsQpTrusted(false);
8335
8336 const int MinEncBitrateKbps = 30;
8337 const int MaxEncBitrateKbps = 100;
8338 const int MinStartBitrateKbp = 50;
8339 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8340 /*frame_size_pixels=*/codec_width_ * codec_height_,
8341 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8342 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8343 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8344
8345 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008346 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008347
8348 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8349
8350 VideoEncoderConfig video_encoder_config;
8351 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8352 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8353 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8354 MinEncBitrateKbps * 1000;
8355 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8356 kMaxPayloadLength);
8357
8358 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8359 WaitForEncodedFrame(1);
8360 EXPECT_EQ(
8361 MaxEncBitrateKbps,
8362 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8363 EXPECT_EQ(
8364 MinEncBitrateKbps,
8365 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8366
8367 video_stream_encoder_->Stop();
8368}
8369
8370TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8371 // Set QP trusted in encoder info.
8372 fake_encoder_.SetIsQpTrusted(false);
8373
8374 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8375 EncoderInfoSettings::
8376 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8377 codec_width_ * codec_height_,
8378 EncoderInfoSettings::
8379 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8380 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8381
8382 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8383 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8384 const int TargetEncBitrate = MaxEncBitrate;
8385 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8386 DataRate::BitsPerSec(TargetEncBitrate),
8387 DataRate::BitsPerSec(TargetEncBitrate),
8388 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8389
8390 VideoEncoderConfig video_encoder_config;
8391 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8392 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8393 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8394 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8395 kMaxPayloadLength);
8396
8397 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8398 WaitForEncodedFrame(1);
8399 EXPECT_EQ(
8400 MaxEncBitrate / 1000,
8401 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8402 EXPECT_EQ(
8403 MinEncBitrate / 1000,
8404 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8405
8406 video_stream_encoder_->Stop();
8407}
8408
Erik Språnge4589cb2022-04-06 16:44:30 +02008409TEST_F(VideoStreamEncoderTest, NormalComplexityWithMoreThanTwoCores) {
8410 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8411 /*num_spatial_layers=*/1,
8412 /*screenshare=*/false, /*allocation_callback_type=*/
8413 VideoStreamEncoder::BitrateAllocationCallbackType::
8414 kVideoBitrateAllocationWhenScreenSharing,
8415 /*num_cores=*/3);
8416
8417 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8418 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8419 video_source_.IncomingCapturedFrame(
8420 CreateFrame(1, /*width=*/320, /*height=*/180));
8421 WaitForEncodedFrame(1);
8422 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8423 VideoCodecComplexity::kComplexityNormal);
8424 video_stream_encoder_->Stop();
8425}
8426
8427TEST_F(VideoStreamEncoderTest,
8428 NormalComplexityWhenLowTierOptimizationsAreDisabled) {
8429 webrtc::test::ScopedKeyValueConfig field_trials(
8430 field_trials_, "WebRTC-VP9-LowTierOptimizations/Disabled/");
8431
8432 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8433 /*num_spatial_layers=*/1,
8434 /*screenshare=*/false, /*allocation_callback_type=*/
8435 VideoStreamEncoder::BitrateAllocationCallbackType::
8436 kVideoBitrateAllocationWhenScreenSharing,
8437 /*num_cores=*/2);
8438
8439 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8440 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8441 video_source_.IncomingCapturedFrame(
8442 CreateFrame(1, /*width=*/320, /*height=*/180));
8443 WaitForEncodedFrame(1);
8444 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8445 VideoCodecComplexity::kComplexityNormal);
8446 video_stream_encoder_->Stop();
8447}
8448
8449TEST_F(VideoStreamEncoderTest, LowComplexityWithTwoCores) {
8450 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8451 /*num_spatial_layers=*/1,
8452 /*screenshare=*/false, /*allocation_callback_type=*/
8453 VideoStreamEncoder::BitrateAllocationCallbackType::
8454 kVideoBitrateAllocationWhenScreenSharing,
8455 /*num_cores=*/2);
8456
8457 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8458 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8459 video_source_.IncomingCapturedFrame(
8460 CreateFrame(1, /*width=*/320, /*height=*/180));
8461 WaitForEncodedFrame(1);
8462 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8463 VideoCodecComplexity::kComplexityLow);
8464 video_stream_encoder_->Stop();
8465}
8466
Sergey Silkind19e3b92021-03-16 10:05:30 +00008467#if !defined(WEBRTC_IOS)
8468// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8469// disabled by default on iOS.
8470TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8471 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8472
8473 // Disable scaling settings in encoder info.
8474 fake_encoder_.SetQualityScaling(false);
8475 // Enable quality scaling in encoder config.
8476 video_encoder_config.is_quality_scaling_allowed = true;
8477 ConfigureEncoder(std::move(video_encoder_config));
8478
8479 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008480 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008481
8482 test::FrameForwarder source;
8483 video_stream_encoder_->SetSource(
8484 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8485 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8486 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8487
8488 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8489 WaitForEncodedFrame(1);
8490 video_stream_encoder_->TriggerQualityLow();
8491 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8492
8493 video_stream_encoder_->Stop();
8494}
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008495
8496TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8497 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8498
8499 // Disable scaling settings in encoder info.
8500 fake_encoder_.SetQualityScaling(false);
8501 // Set QP trusted in encoder info.
8502 fake_encoder_.SetIsQpTrusted(true);
8503 // Enable quality scaling in encoder config.
8504 video_encoder_config.is_quality_scaling_allowed = true;
8505 ConfigureEncoder(std::move(video_encoder_config));
8506
8507 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008508 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008509
8510 test::FrameForwarder source;
8511 video_stream_encoder_->SetSource(
8512 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8513 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8514 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8515
8516 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8517 WaitForEncodedFrame(1);
8518 video_stream_encoder_->TriggerQualityLow();
8519 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8520
8521 video_stream_encoder_->Stop();
8522}
Shuhai Pengf2707702021-09-29 17:19:44 +08008523
8524TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8525 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8526
8527 // Disable scaling settings in encoder info.
8528 fake_encoder_.SetQualityScaling(false);
8529 // Set QP not trusted in encoder info.
8530 fake_encoder_.SetIsQpTrusted(false);
8531 // Enable quality scaling in encoder config.
8532 video_encoder_config.is_quality_scaling_allowed = true;
8533 ConfigureEncoder(std::move(video_encoder_config));
8534
8535 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008536 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008537
8538 test::FrameForwarder source;
8539 video_stream_encoder_->SetSource(
8540 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8541 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8542 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8543
8544 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8545 WaitForEncodedFrame(1);
8546 video_stream_encoder_->TriggerQualityLow();
8547 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8548 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8549 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8550
8551 video_stream_encoder_->Stop();
8552}
8553
8554TEST_F(VideoStreamEncoderTest,
8555 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8556 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8557
8558 // Disable scaling settings in encoder info.
8559 fake_encoder_.SetQualityScaling(false);
8560 // Set QP trusted in encoder info.
8561 fake_encoder_.SetIsQpTrusted(true);
8562 // Enable quality scaling in encoder config.
8563 video_encoder_config.is_quality_scaling_allowed = true;
8564 ConfigureEncoder(std::move(video_encoder_config));
8565
8566 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008567 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008568
8569 test::FrameForwarder source;
8570 video_stream_encoder_->SetSource(
8571 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8572 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8573 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8574
8575 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8576 WaitForEncodedFrame(1);
8577 video_stream_encoder_->TriggerQualityLow();
8578 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8579 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8580
8581 video_stream_encoder_->Stop();
8582}
8583
8584TEST_F(VideoStreamEncoderTest,
8585 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8586 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8587
8588 // Disable scaling settings in encoder info.
8589 fake_encoder_.SetQualityScaling(false);
8590 // Set QP trusted in encoder info.
8591 fake_encoder_.SetIsQpTrusted(false);
8592 // Enable quality scaling in encoder config.
8593 video_encoder_config.is_quality_scaling_allowed = true;
8594 ConfigureEncoder(std::move(video_encoder_config));
8595
8596 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008597 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008598
8599 test::FrameForwarder source;
8600 video_stream_encoder_->SetSource(
8601 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8602 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8603 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8604
8605 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8606 WaitForEncodedFrame(1);
8607 video_stream_encoder_->TriggerQualityLow();
8608 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8609
8610 video_stream_encoder_->Stop();
8611}
8612
Erik Språnge4589cb2022-04-06 16:44:30 +02008613#endif // !defined(WEBRTC_IOS)
Sergey Silkind19e3b92021-03-16 10:05:30 +00008614
Henrik Boström56db9ff2021-03-24 09:06:45 +01008615// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8616class VideoStreamEncoderWithRealEncoderTest
8617 : public VideoStreamEncoderTest,
8618 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8619 public:
8620 VideoStreamEncoderWithRealEncoderTest()
8621 : VideoStreamEncoderTest(),
8622 codec_type_(std::get<0>(GetParam())),
8623 allow_i420_conversion_(std::get<1>(GetParam())) {}
8624
8625 void SetUp() override {
8626 VideoStreamEncoderTest::SetUp();
8627 std::unique_ptr<VideoEncoder> encoder;
8628 switch (codec_type_) {
8629 case kVideoCodecVP8:
8630 encoder = VP8Encoder::Create();
8631 break;
8632 case kVideoCodecVP9:
8633 encoder = VP9Encoder::Create();
8634 break;
8635 case kVideoCodecAV1:
philipel09a28482022-05-25 09:47:06 +02008636 encoder = CreateLibaomAv1Encoder();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008637 break;
8638 case kVideoCodecH264:
8639 encoder =
8640 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8641 break;
8642 case kVideoCodecMultiplex:
8643 mock_encoder_factory_for_multiplex_ =
8644 std::make_unique<MockVideoEncoderFactory>();
8645 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8646 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8647 .WillRepeatedly([] { return VP8Encoder::Create(); });
8648 encoder = std::make_unique<MultiplexEncoderAdapter>(
8649 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8650 false);
8651 break;
8652 default:
Artem Titovd3251962021-11-15 16:57:07 +01008653 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008654 }
8655 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8656 }
8657
8658 void TearDown() override {
8659 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 16:22:11 +02008660 // Ensure `video_stream_encoder_` is destroyed before
8661 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 09:06:45 +01008662 video_stream_encoder_.reset();
8663 VideoStreamEncoderTest::TearDown();
8664 }
8665
8666 protected:
8667 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8668 std::unique_ptr<VideoEncoder> encoder) {
8669 // Configure VSE to use the encoder.
8670 encoder_ = std::move(encoder);
8671 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8672 encoder_.get(), &encoder_selector_);
8673 video_send_config_.encoder_settings.encoder_factory =
8674 encoder_proxy_factory_.get();
8675 VideoEncoderConfig video_encoder_config;
8676 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8677 video_encoder_config_ = video_encoder_config.Copy();
8678 ConfigureEncoder(video_encoder_config_.Copy());
8679
8680 // Set bitrate to ensure frame is not dropped.
8681 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008682 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008683 }
8684
8685 const VideoCodecType codec_type_;
8686 const bool allow_i420_conversion_;
8687 NiceMock<MockEncoderSelector> encoder_selector_;
8688 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8689 std::unique_ptr<VideoEncoder> encoder_;
8690 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8691};
8692
8693TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8694 auto native_i420_frame = test::CreateMappableNativeFrame(
8695 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8696 video_source_.IncomingCapturedFrame(native_i420_frame);
8697 WaitForEncodedFrame(codec_width_, codec_height_);
8698
8699 auto mappable_native_buffer =
8700 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8701 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8702 mappable_native_buffer->GetMappedFramedBuffers();
8703 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8704 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8705 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8706 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8707}
8708
8709TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8710 auto native_nv12_frame = test::CreateMappableNativeFrame(
8711 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8712 video_source_.IncomingCapturedFrame(native_nv12_frame);
8713 WaitForEncodedFrame(codec_width_, codec_height_);
8714
8715 auto mappable_native_buffer =
8716 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8717 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8718 mappable_native_buffer->GetMappedFramedBuffers();
8719 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8720 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8721 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8722 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8723
8724 if (!allow_i420_conversion_) {
8725 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8726 }
8727}
8728
Erik Språng7444b192021-06-02 14:02:13 +02008729TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8730 if (codec_type_ == kVideoCodecMultiplex) {
8731 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8732 return;
8733 }
8734
8735 const size_t kNumSpatialLayers = 3u;
8736 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8737 const int kFrameWidth = 1280;
8738 const int kFrameHeight = 720;
8739 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8740 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8741 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8742 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8743 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8744 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8745
8746 VideoEncoderConfig config;
8747 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8748 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008749 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008750 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8751 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8752 vp9_settings.numberOfTemporalLayers = 3;
8753 vp9_settings.automaticResizeOn = false;
8754 config.encoder_specific_settings =
8755 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8756 vp9_settings);
8757 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8758 /*fps=*/30.0,
8759 /*first_active_layer=*/0,
8760 /*num_spatial_layers=*/3,
8761 /*num_temporal_layers=*/3,
8762 /*is_screenshare=*/false);
8763 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8764 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008765 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008766 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8767 /*fps=*/30.0,
8768 /*first_active_layer=*/0,
8769 /*num_spatial_layers=*/3,
8770 /*num_temporal_layers=*/3,
8771 /*is_screenshare=*/false);
Niels Möller79d566b2022-04-29 11:03:13 +02008772 config.simulcast_layers[0].scalability_mode = ScalabilityMode::kL3T3_KEY;
Erik Språng7444b192021-06-02 14:02:13 +02008773 } else {
8774 // Simulcast for VP8/H264.
8775 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8776 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8777 config.simulcast_layers[i].scale_resolution_down_by =
8778 kDownscaleFactors[i];
8779 config.simulcast_layers[i].active = true;
8780 }
8781 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8782 // Turn off frame dropping to prevent flakiness.
Niels Möller807328f2022-05-12 16:16:39 +02008783 config.frame_drop_enabled = false;
Erik Språng7444b192021-06-02 14:02:13 +02008784 }
8785 }
8786
8787 auto set_layer_active = [&](int layer_idx, bool active) {
8788 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8789 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8790 config.spatial_layers[layer_idx].active = active;
8791 } else {
8792 config.simulcast_layers[layer_idx].active = active;
8793 }
8794 };
8795
8796 config.video_stream_factory =
8797 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8798 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8799 /*screencast*/ false,
8800 /*screenshare enabled*/ false);
8801 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008802 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8803 0, 0, 0);
Erik Språng7444b192021-06-02 14:02:13 +02008804
8805 // Capture a frame with all layers active.
8806 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8807 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8808 int64_t timestamp_ms = kFrameIntervalMs;
8809 video_source_.IncomingCapturedFrame(
8810 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8811
8812 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8813 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8814
8815 // Capture a frame with one of the layers inactive.
8816 set_layer_active(2, false);
8817 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8818 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8819 timestamp_ms += kFrameIntervalMs;
8820 video_source_.IncomingCapturedFrame(
8821 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8822 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8823
8824 // New target bitrates signaled based on lower resolution.
8825 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8826 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8827 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8828 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8829
8830 // Re-enable the top layer.
8831 set_layer_active(2, true);
8832 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8833 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8834 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8835
8836 // Bitrate target adjusted back up to enable HD layer...
8837 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8838 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8839 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8840 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8841
8842 // ...then add a new frame.
8843 timestamp_ms += kFrameIntervalMs;
8844 video_source_.IncomingCapturedFrame(
8845 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8846 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8847 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8848
8849 video_stream_encoder_->Stop();
8850}
8851
Henrik Boström56db9ff2021-03-24 09:06:45 +01008852std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8853 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8854 VideoCodecType codec_type = std::get<0>(info.param);
8855 bool allow_i420_conversion = std::get<1>(info.param);
8856 std::string str;
8857 switch (codec_type) {
8858 case kVideoCodecGeneric:
8859 str = "Generic";
8860 break;
8861 case kVideoCodecVP8:
8862 str = "VP8";
8863 break;
8864 case kVideoCodecVP9:
8865 str = "VP9";
8866 break;
8867 case kVideoCodecAV1:
8868 str = "AV1";
8869 break;
8870 case kVideoCodecH264:
8871 str = "H264";
8872 break;
8873 case kVideoCodecMultiplex:
8874 str = "Multiplex";
8875 break;
8876 default:
Artem Titovd3251962021-11-15 16:57:07 +01008877 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008878 }
8879 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8880 return str;
8881}
8882
8883constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8884 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8885constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8886 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8887constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
Ilya Nikolaevskiy1bcdafc2022-03-09 16:01:07 +01008888 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/false);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008889constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8890 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8891#if defined(WEBRTC_USE_H264)
8892constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8893 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8894
8895// The windows compiler does not tolerate #if statements inside the
8896// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8897// and without H264).
8898INSTANTIATE_TEST_SUITE_P(
8899 All,
8900 VideoStreamEncoderWithRealEncoderTest,
8901 ::testing::Values(kVP8DisallowConversion,
8902 kVP9DisallowConversion,
8903 kAV1AllowConversion,
8904 kMultiplexDisallowConversion,
8905 kH264AllowConversion),
8906 TestParametersVideoCodecAndAllowI420ConversionToString);
8907#else
8908INSTANTIATE_TEST_SUITE_P(
8909 All,
8910 VideoStreamEncoderWithRealEncoderTest,
8911 ::testing::Values(kVP8DisallowConversion,
8912 kVP9DisallowConversion,
8913 kAV1AllowConversion,
8914 kMultiplexDisallowConversion),
8915 TestParametersVideoCodecAndAllowI420ConversionToString);
8916#endif
8917
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008918class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8919 protected:
8920 void RunTest(const std::vector<VideoStream>& configs,
8921 const int expected_num_init_encode) {
8922 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008923 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008924 InsertFrameAndWaitForEncoded();
8925 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8926 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008927 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8928 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008929
8930 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8931 ConfigureEncoder(configs[1]);
8932 InsertFrameAndWaitForEncoded();
8933 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8934 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008935 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008936 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 10:07:11 +02008937 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008938
8939 video_stream_encoder_->Stop();
8940 }
8941
8942 void ConfigureEncoder(const VideoStream& stream) {
8943 VideoEncoderConfig config;
8944 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8945 config.max_bitrate_bps = stream.max_bitrate_bps;
8946 config.simulcast_layers[0] = stream;
8947 config.video_stream_factory =
8948 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8949 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8950 /*conference_mode=*/false);
8951 video_stream_encoder_->ConfigureEncoder(std::move(config),
8952 kMaxPayloadLength);
8953 }
8954
8955 void OnBitrateUpdated(DataRate bitrate) {
8956 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8957 bitrate, bitrate, bitrate, 0, 0, 0);
8958 }
8959
8960 void InsertFrameAndWaitForEncoded() {
8961 timestamp_ms_ += kFrameIntervalMs;
8962 video_source_.IncomingCapturedFrame(
8963 CreateFrame(timestamp_ms_, kWidth, kHeight));
8964 sink_.WaitForEncodedFrame(timestamp_ms_);
8965 }
8966
8967 void ExpectEqual(const VideoCodec& actual,
8968 const VideoStream& expected) const {
8969 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8970 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8971 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
8972 static_cast<unsigned int>(expected.min_bitrate_bps));
8973 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
8974 static_cast<unsigned int>(expected.max_bitrate_bps));
8975 EXPECT_EQ(actual.simulcastStream[0].width,
8976 kWidth / expected.scale_resolution_down_by);
8977 EXPECT_EQ(actual.simulcastStream[0].height,
8978 kHeight / expected.scale_resolution_down_by);
8979 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
8980 expected.num_temporal_layers);
Niels Möller79d566b2022-04-29 11:03:13 +02008981 EXPECT_EQ(actual.GetScalabilityMode(), expected.scalability_mode);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008982 }
8983
8984 VideoStream DefaultConfig() const {
8985 VideoStream stream;
8986 stream.max_framerate = 25;
8987 stream.min_bitrate_bps = 35000;
8988 stream.max_bitrate_bps = 900000;
8989 stream.scale_resolution_down_by = 1.0;
8990 stream.num_temporal_layers = 1;
8991 stream.bitrate_priority = 1.0;
Niels Möller79d566b2022-04-29 11:03:13 +02008992 stream.scalability_mode = absl::nullopt;
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008993 return stream;
8994 }
8995
8996 const int kWidth = 640;
8997 const int kHeight = 360;
8998 int64_t timestamp_ms_ = 0;
8999};
9000
9001TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
9002 VideoStream config1 = DefaultConfig();
9003 VideoStream config2 = config1;
9004 config2.max_framerate++;
9005
9006 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9007}
9008
9009TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
9010 VideoStream config1 = DefaultConfig();
9011 VideoStream config2 = config1;
9012 config2.min_bitrate_bps += 10000;
9013
9014 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9015}
9016
9017TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
9018 VideoStream config1 = DefaultConfig();
9019 VideoStream config2 = config1;
9020 config2.max_bitrate_bps += 100000;
9021
9022 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9023}
9024
9025TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
9026 VideoStream config1 = DefaultConfig();
9027 VideoStream config2 = config1;
9028 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
9029
9030 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9031}
9032
9033TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
9034 VideoStream config1 = DefaultConfig();
9035 VideoStream config2 = config1;
9036 config2.scale_resolution_down_by *= 2;
9037
9038 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9039}
9040
9041TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
9042 VideoStream config1 = DefaultConfig();
9043 VideoStream config2 = config1;
9044 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
9045
9046 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9047}
9048
9049TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
9050 VideoStream config1 = DefaultConfig();
9051 VideoStream config2 = config1;
Niels Möller05954892022-05-13 13:34:37 +02009052 config2.scalability_mode = ScalabilityMode::kL2T1;
9053
9054 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9055}
9056
9057TEST_F(ReconfigureEncoderTest,
9058 UpdatesNumTemporalLayersFromScalabilityModeChanges) {
9059 VideoStream config1 = DefaultConfig();
9060 VideoStream config2 = config1;
Niels Möller79d566b2022-04-29 11:03:13 +02009061 config2.scalability_mode = ScalabilityMode::kL1T2;
Niels Möller05954892022-05-13 13:34:37 +02009062 config2.num_temporal_layers = 2;
Åsa Persson4d4f62f2021-09-12 16:14:48 +02009063
9064 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9065}
9066
Tommi62b01db2022-01-25 23:41:22 +01009067// Simple test that just creates and then immediately destroys an encoder.
9068// The purpose of the test is to make sure that nothing bad happens if the
9069// initialization step on the encoder queue, doesn't run.
9070TEST(VideoStreamEncoderSimpleTest, CreateDestroy) {
9071 class SuperLazyTaskQueue : public webrtc::TaskQueueBase {
9072 public:
9073 SuperLazyTaskQueue() = default;
9074 ~SuperLazyTaskQueue() override = default;
9075
9076 private:
9077 void Delete() override { delete this; }
Danil Chapovalov95eeaa72022-07-06 10:14:29 +02009078 void PostTask(absl::AnyInvocable<void() &&> task) override {
Tommi62b01db2022-01-25 23:41:22 +01009079 // meh.
9080 }
Danil Chapovalov95eeaa72022-07-06 10:14:29 +02009081 void PostDelayedTask(absl::AnyInvocable<void() &&> task,
9082 TimeDelta delay) override {
Tommi62b01db2022-01-25 23:41:22 +01009083 ASSERT_TRUE(false);
9084 }
Danil Chapovalov95eeaa72022-07-06 10:14:29 +02009085 void PostDelayedHighPrecisionTask(absl::AnyInvocable<void() &&> task,
9086 TimeDelta delay) override {
9087 ADD_FAILURE();
9088 }
Tommi62b01db2022-01-25 23:41:22 +01009089 };
9090
9091 // Lots of boiler plate.
Jonas Oreland8ca06132022-03-14 12:52:48 +01009092 test::ScopedKeyValueConfig field_trials;
Tommi62b01db2022-01-25 23:41:22 +01009093 GlobalSimulatedTimeController time_controller(Timestamp::Millis(0));
9094 auto stats_proxy = std::make_unique<MockableSendStatisticsProxy>(
9095 time_controller.GetClock(), VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 12:52:48 +01009096 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo, field_trials);
Tommi62b01db2022-01-25 23:41:22 +01009097 SimpleVideoStreamEncoderFactory::MockFakeEncoder mock_fake_encoder(
9098 time_controller.GetClock());
9099 test::VideoEncoderProxyFactory encoder_factory(&mock_fake_encoder);
9100 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory =
9101 CreateBuiltinVideoBitrateAllocatorFactory();
9102 VideoStreamEncoderSettings encoder_settings{
9103 VideoEncoder::Capabilities(/*loss_notification=*/false)};
9104 encoder_settings.encoder_factory = &encoder_factory;
9105 encoder_settings.bitrate_allocator_factory = bitrate_allocator_factory.get();
9106
9107 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9108 EXPECT_CALL((*adapter.get()), Initialize).WillOnce(Return());
9109
9110 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
9111 encoder_queue(new SuperLazyTaskQueue());
9112
9113 // Construct a VideoStreamEncoder instance and let it go out of scope without
9114 // doing anything else (including calling Stop()). This should be fine since
9115 // the posted init task will simply be deleted.
9116 auto encoder = std::make_unique<VideoStreamEncoder>(
9117 time_controller.GetClock(), 1, stats_proxy.get(), encoder_settings,
Markus Handell8e4197b2022-05-30 15:45:28 +02009118 std::make_unique<CpuOveruseDetectorProxy>(stats_proxy.get(),
9119 field_trials),
Tommi62b01db2022-01-25 23:41:22 +01009120 std::move(adapter), std::move(encoder_queue),
9121 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009122 kVideoBitrateAllocation,
9123 field_trials);
Tommi4d6dd172022-05-23 09:34:41 +02009124
9125 // Stop the encoder explicitly. This additional step tests if we could
9126 // hang when calling stop and the TQ has been stopped and/or isn't accepting
9127 // any more tasks.
9128 encoder->Stop();
Tommi62b01db2022-01-25 23:41:22 +01009129}
9130
Markus Handellb4e96d42021-11-05 12:00:55 +01009131TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
9132 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9133 auto* adapter_ptr = adapter.get();
9134 SimpleVideoStreamEncoderFactory factory;
Markus Handell8d87c462021-12-16 11:37:16 +01009135 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9136 nullptr;
9137 EXPECT_CALL(*adapter_ptr, Initialize)
9138 .WillOnce(Invoke([&video_stream_encoder_callback](
9139 FrameCadenceAdapterInterface::Callback* callback) {
9140 video_stream_encoder_callback = callback;
9141 }));
9142 TaskQueueBase* encoder_queue = nullptr;
9143 auto video_stream_encoder =
9144 factory.Create(std::move(adapter), &encoder_queue);
Markus Handellb4e96d42021-11-05 12:00:55 +01009145
Markus Handelle59fee82021-12-23 09:29:23 +01009146 // First a call before we know the frame size and hence cannot compute the
9147 // number of simulcast layers.
9148 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
9149 &FrameCadenceAdapterInterface::
9150 ZeroHertzModeParams::num_simulcast_layers,
9151 Eq(0)))));
Markus Handellb4e96d42021-11-05 12:00:55 +01009152 VideoEncoderConfig config;
Markus Handell8d87c462021-12-16 11:37:16 +01009153 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
Markus Handellb4e96d42021-11-05 12:00:55 +01009154 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9155 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
Markus Handelle59fee82021-12-23 09:29:23 +01009156 factory.DepleteTaskQueues();
9157
9158 // Then a call as we've computed the number of simulcast layers after a passed
9159 // frame.
9160 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
9161 &FrameCadenceAdapterInterface::
9162 ZeroHertzModeParams::num_simulcast_layers,
9163 Gt(0)))));
Markus Handell8d87c462021-12-16 11:37:16 +01009164 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handell9a478b52021-11-18 16:07:01 +01009165 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01009166 Mock::VerifyAndClearExpectations(adapter_ptr);
9167
Markus Handelle59fee82021-12-23 09:29:23 +01009168 // Expect a disabled zero-hertz mode after passing realtime video.
Markus Handell8d87c462021-12-16 11:37:16 +01009169 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Eq(absl::nullopt)));
Markus Handellb4e96d42021-11-05 12:00:55 +01009170 VideoEncoderConfig config2;
Markus Handell8d87c462021-12-16 11:37:16 +01009171 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config2);
Markus Handellb4e96d42021-11-05 12:00:55 +01009172 config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
9173 video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
Markus Handell8d87c462021-12-16 11:37:16 +01009174 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
Markus Handell9a478b52021-11-18 16:07:01 +01009175 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01009176}
9177
9178TEST(VideoStreamEncoderFrameCadenceTest,
9179 ForwardsFramesIntoFrameCadenceAdapter) {
9180 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9181 auto* adapter_ptr = adapter.get();
9182 test::FrameForwarder video_source;
9183 SimpleVideoStreamEncoderFactory factory;
9184 auto video_stream_encoder = factory.Create(std::move(adapter));
9185 video_stream_encoder->SetSource(
9186 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9187
9188 EXPECT_CALL(*adapter_ptr, OnFrame);
9189 auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
9190 video_source.IncomingCapturedFrame(
Markus Handell8d87c462021-12-16 11:37:16 +01009191 VideoFrame::Builder().set_video_frame_buffer(std::move(buffer)).build());
Markus Handellb4e96d42021-11-05 12:00:55 +01009192}
9193
Markus Handellee225432021-11-29 12:35:12 +01009194TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) {
9195 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9196 auto* adapter_ptr = adapter.get();
9197 test::FrameForwarder video_source;
9198 SimpleVideoStreamEncoderFactory factory;
9199 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9200 nullptr;
9201 EXPECT_CALL(*adapter_ptr, Initialize)
9202 .WillOnce(Invoke([&video_stream_encoder_callback](
9203 FrameCadenceAdapterInterface::Callback* callback) {
9204 video_stream_encoder_callback = callback;
9205 }));
9206 TaskQueueBase* encoder_queue = nullptr;
9207 auto video_stream_encoder =
9208 factory.Create(std::move(adapter), &encoder_queue);
9209
9210 // This is just to make the VSE operational. We'll feed a frame directly by
9211 // the callback interface.
9212 video_stream_encoder->SetSource(
9213 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9214
9215 VideoEncoderConfig video_encoder_config;
9216 test::FillEncoderConfiguration(kVideoCodecGeneric, 1, &video_encoder_config);
9217 video_stream_encoder->ConfigureEncoder(std::move(video_encoder_config),
9218 /*max_data_payload_length=*/1000);
9219
9220 EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps);
9221 EXPECT_CALL(*adapter_ptr, UpdateFrameRate);
Markus Handell8d87c462021-12-16 11:37:16 +01009222 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handellee225432021-11-29 12:35:12 +01009223 factory.DepleteTaskQueues();
9224}
9225
Markus Handell8d87c462021-12-16 11:37:16 +01009226TEST(VideoStreamEncoderFrameCadenceTest,
9227 DeactivatesActivatesLayersOnBitrateChanges) {
9228 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9229 auto* adapter_ptr = adapter.get();
9230 SimpleVideoStreamEncoderFactory factory;
9231 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9232 nullptr;
9233 EXPECT_CALL(*adapter_ptr, Initialize)
9234 .WillOnce(Invoke([&video_stream_encoder_callback](
9235 FrameCadenceAdapterInterface::Callback* callback) {
9236 video_stream_encoder_callback = callback;
9237 }));
9238 TaskQueueBase* encoder_queue = nullptr;
9239 auto video_stream_encoder =
9240 factory.Create(std::move(adapter), &encoder_queue);
9241
9242 // Configure 2 simulcast layers. FillEncoderConfiguration sets min bitrates to
9243 // {150000, 450000}.
9244 VideoEncoderConfig video_encoder_config;
9245 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9246 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9247 kMaxPayloadLength);
9248 // Ensure an encoder is created.
9249 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9250
9251 // Both layers enabled at 1 MBit/s.
9252 video_stream_encoder->OnBitrateUpdated(
9253 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9254 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9255 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9256 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9257 factory.DepleteTaskQueues();
9258 Mock::VerifyAndClearExpectations(adapter_ptr);
9259
9260 // Layer 1 disabled at 200 KBit/s.
9261 video_stream_encoder->OnBitrateUpdated(
9262 DataRate::KilobitsPerSec(200), DataRate::KilobitsPerSec(200),
9263 DataRate::KilobitsPerSec(200), 0, 0, 0);
9264 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9265 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9266 factory.DepleteTaskQueues();
9267 Mock::VerifyAndClearExpectations(adapter_ptr);
9268
9269 // All layers off at suspended video.
9270 video_stream_encoder->OnBitrateUpdated(DataRate::Zero(), DataRate::Zero(),
9271 DataRate::Zero(), 0, 0, 0);
9272 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/false));
9273 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9274 factory.DepleteTaskQueues();
9275 Mock::VerifyAndClearExpectations(adapter_ptr);
9276
9277 // Both layers enabled again back at 1 MBit/s.
9278 video_stream_encoder->OnBitrateUpdated(
9279 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9280 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9281 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9282 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9283 factory.DepleteTaskQueues();
9284}
9285
9286TEST(VideoStreamEncoderFrameCadenceTest, UpdatesQualityConvergence) {
9287 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9288 auto* adapter_ptr = adapter.get();
9289 SimpleVideoStreamEncoderFactory factory;
9290 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9291 nullptr;
9292 EXPECT_CALL(*adapter_ptr, Initialize)
9293 .WillOnce(Invoke([&video_stream_encoder_callback](
9294 FrameCadenceAdapterInterface::Callback* callback) {
9295 video_stream_encoder_callback = callback;
9296 }));
9297 TaskQueueBase* encoder_queue = nullptr;
9298 auto video_stream_encoder =
9299 factory.Create(std::move(adapter), &encoder_queue);
9300
9301 // Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder.
9302 VideoEncoderConfig video_encoder_config;
9303 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9304 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9305 kMaxPayloadLength);
9306 video_stream_encoder->OnBitrateUpdated(
9307 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9308 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9309
9310 // Pass a frame which has unconverged results.
9311 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9312 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9313 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9314 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
9315 EXPECT_FALSE(encoded_image.IsAtTargetQuality());
9316 CodecSpecificInfo codec_specific;
9317 codec_specific.codecType = kVideoCodecGeneric;
9318 return codec_specific;
9319 }));
9320 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, false));
9321 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9322 factory.DepleteTaskQueues();
9323 Mock::VerifyAndClearExpectations(adapter_ptr);
9324 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9325
9326 // Pass a frame which converges in layer 0 and not in layer 1.
9327 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9328 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9329 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9330 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
9331 encoded_image.SetAtTargetQuality(encoded_image.SpatialIndex() == 0);
9332 CodecSpecificInfo codec_specific;
9333 codec_specific.codecType = kVideoCodecGeneric;
9334 return codec_specific;
9335 }));
9336 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, true));
9337 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9338 factory.DepleteTaskQueues();
9339 Mock::VerifyAndClearExpectations(adapter_ptr);
9340 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9341}
9342
Markus Handell2e0f4f02021-12-21 19:14:58 +01009343TEST(VideoStreamEncoderFrameCadenceTest,
9344 RequestsRefreshFramesWhenCadenceAdapterInstructs) {
9345 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9346 auto* adapter_ptr = adapter.get();
9347 MockVideoSourceInterface mock_source;
9348 SimpleVideoStreamEncoderFactory factory;
9349 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9350 nullptr;
9351 EXPECT_CALL(*adapter_ptr, Initialize)
9352 .WillOnce(Invoke([&video_stream_encoder_callback](
9353 FrameCadenceAdapterInterface::Callback* callback) {
9354 video_stream_encoder_callback = callback;
9355 }));
9356 TaskQueueBase* encoder_queue = nullptr;
9357 auto video_stream_encoder =
9358 factory.Create(std::move(adapter), &encoder_queue);
9359 video_stream_encoder->SetSource(
9360 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9361 VideoEncoderConfig config;
9362 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9363 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9364 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9365 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9366 // Ensure the encoder is set up.
9367 factory.DepleteTaskQueues();
9368
Markus Handell818e7fb2021-12-30 13:01:33 +01009369 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest)
9370 .WillOnce(Invoke([video_stream_encoder_callback] {
9371 video_stream_encoder_callback->RequestRefreshFrame();
9372 }));
Markus Handell2e0f4f02021-12-21 19:14:58 +01009373 EXPECT_CALL(mock_source, RequestRefreshFrame);
9374 video_stream_encoder->SendKeyFrame();
9375 factory.DepleteTaskQueues();
9376 Mock::VerifyAndClearExpectations(adapter_ptr);
9377 Mock::VerifyAndClearExpectations(&mock_source);
9378
Markus Handell818e7fb2021-12-30 13:01:33 +01009379 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest);
Markus Handell2e0f4f02021-12-21 19:14:58 +01009380 EXPECT_CALL(mock_source, RequestRefreshFrame).Times(0);
9381 video_stream_encoder->SendKeyFrame();
9382 factory.DepleteTaskQueues();
9383}
9384
Markus Handell818e7fb2021-12-30 13:01:33 +01009385TEST(VideoStreamEncoderFrameCadenceTest,
9386 RequestsRefreshFrameForEarlyZeroHertzKeyFrameRequest) {
9387 SimpleVideoStreamEncoderFactory factory;
9388 auto encoder_queue =
9389 factory.GetTimeController()->GetTaskQueueFactory()->CreateTaskQueue(
9390 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
9391
9392 // Enables zero-hertz mode.
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009393 test::ScopedKeyValueConfig field_trials(
9394 "WebRTC-ZeroHertzScreenshare/Enabled/");
Markus Handell818e7fb2021-12-30 13:01:33 +01009395 auto adapter = FrameCadenceAdapterInterface::Create(
Jonas Oreland8ca06132022-03-14 12:52:48 +01009396 factory.GetTimeController()->GetClock(), encoder_queue.get(),
9397 field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009398 FrameCadenceAdapterInterface* adapter_ptr = adapter.get();
9399
9400 MockVideoSourceInterface mock_source;
9401 auto video_stream_encoder = factory.CreateWithEncoderQueue(
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009402 std::move(adapter), std::move(encoder_queue), &field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009403
9404 video_stream_encoder->SetSource(
9405 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9406 VideoEncoderConfig config;
9407 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9408 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9409 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9410
9411 // Eventually expect a refresh frame request when requesting a key frame
9412 // before initializing zero-hertz mode. This can happen in reality because the
9413 // threads invoking key frame requests and constraints setup aren't
9414 // synchronized.
9415 EXPECT_CALL(mock_source, RequestRefreshFrame);
9416 video_stream_encoder->SendKeyFrame();
Markus Handellf5a50792022-06-16 14:25:15 +02009417 constexpr int kMaxFps = 30;
9418 adapter_ptr->OnConstraintsChanged(VideoTrackSourceConstraints{0, kMaxFps});
9419 factory.GetTimeController()->AdvanceTime(
9420 TimeDelta::Seconds(1) *
9421 FrameCadenceAdapterInterface::kOnDiscardedFrameRefreshFramePeriod /
9422 kMaxFps);
Markus Handell818e7fb2021-12-30 13:01:33 +01009423}
9424
perkj26091b12016-09-01 01:17:40 -07009425} // namespace webrtc