blob: 190c1d7ab204f8e8c621c38f76a29dedbb9d6079 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Markus Handell8e4197b2022-05-30 15:45:28 +020020#include "api/field_trials_view.h"
Markus Handell9a478b52021-11-18 16:07:01 +010021#include "api/rtp_parameters.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020022#include "api/task_queue/default_task_queue_factory.h"
Markus Handell818e7fb2021-12-30 13:01:33 +010023#include "api/task_queue/task_queue_base.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010024#include "api/task_queue/task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020025#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010026#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010027#include "api/test/mock_video_encoder_factory.h"
Markus Handell8d87c462021-12-16 11:37:16 +010028#include "api/units/data_rate.h"
Markus Handellf5a50792022-06-16 14:25:15 +020029#include "api/units/time_delta.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080030#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020031#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020032#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020033#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010034#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010035#include "api/video_codecs/sdp_video_format.h"
Erik Språnge4589cb2022-04-06 16:44:30 +020036#include "api/video_codecs/video_codec.h"
Elad Alon370f93a2019-06-11 14:57:57 +020037#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020038#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010039#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020040#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010041#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020042#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070043#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080044#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020045#include "media/engine/webrtc_video_engine.h"
philipel09a28482022-05-25 09:47:06 +020046#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010047#include "modules/video_coding/codecs/h264/include/h264.h"
48#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
49#include "modules/video_coding/codecs/vp8/include/vp8.h"
50#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020051#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020052#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020053#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010054#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020055#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010056#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020057#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020058#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080059#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020060#include "rtc_base/synchronization/mutex.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020061#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020062#include "test/encoder_settings.h"
63#include "test/fake_encoder.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010064#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020065#include "test/gmock.h"
66#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010067#include "test/mappable_native_buffer.h"
Jonas Orelandc7f691a2022-03-09 15:12:07 +010068#include "test/scoped_key_value_config.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020069#include "test/time_controller/simulated_time_controller.h"
Byoungchan Lee13fe3672022-04-06 10:44:42 +090070#include "test/video_encoder_nullable_proxy_factory.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020071#include "test/video_encoder_proxy_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010072#include "video/frame_cadence_adapter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020073#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070074
75namespace webrtc {
76
sprang57c2fff2017-01-16 06:24:02 -080077using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020078using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020079using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020080using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020081using ::testing::Ge;
82using ::testing::Gt;
Markus Handellee225432021-11-29 12:35:12 +010083using ::testing::Invoke;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020084using ::testing::Le;
85using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010086using ::testing::Matcher;
Markus Handellb4e96d42021-11-05 12:00:55 +010087using ::testing::Mock;
philipel9b058032020-02-10 11:30:00 +010088using ::testing::NiceMock;
Markus Handellb4e96d42021-11-05 12:00:55 +010089using ::testing::Optional;
philipel9b058032020-02-10 11:30:00 +010090using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010091using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020092using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080093
perkj803d97f2016-11-01 11:45:46 -070094namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020095const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010096const int kQpLow = 1;
97const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020098const int kMinFramerateFps = 2;
99const int kMinBalancedFramerateFps = 7;
100const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -0800101const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 10:07:11 +0200102const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
103const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
104const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
105const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 07:02:22 -0800106const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -0700107const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200108const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +0200109const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200110const VideoEncoder::ResolutionBitrateLimits
111 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
112const VideoEncoder::ResolutionBitrateLimits
113 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800114
Asa Persson606d3cb2021-10-04 10:07:11 +0200115uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200116 0x00, 0x00, 0x03, 0x03, 0xF4,
117 0x05, 0x03, 0xC7, 0xE0, 0x1B,
118 0x41, 0x10, 0x8D, 0x00};
119
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100120const uint8_t kCodedFrameVp8Qp25[] = {
121 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
122 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
123 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
124
Markus Handell818e7fb2021-12-30 13:01:33 +0100125VideoFrame CreateSimpleNV12Frame() {
126 return VideoFrame::Builder()
127 .set_video_frame_buffer(rtc::make_ref_counted<NV12Buffer>(
128 /*width=*/16, /*height=*/16))
129 .build();
130}
131
Markus Handell8d87c462021-12-16 11:37:16 +0100132void PassAFrame(
133 TaskQueueBase* encoder_queue,
134 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback,
135 int64_t ntp_time_ms) {
Danil Chapovalov95eeaa72022-07-06 10:14:29 +0200136 encoder_queue->PostTask([video_stream_encoder_callback, ntp_time_ms] {
137 video_stream_encoder_callback->OnFrame(Timestamp::Millis(ntp_time_ms), 1,
138 CreateSimpleNV12Frame());
139 });
Markus Handell8d87c462021-12-16 11:37:16 +0100140}
141
perkj803d97f2016-11-01 11:45:46 -0700142class TestBuffer : public webrtc::I420Buffer {
143 public:
144 TestBuffer(rtc::Event* event, int width, int height)
145 : I420Buffer(width, height), event_(event) {}
146
147 private:
148 friend class rtc::RefCountedObject<TestBuffer>;
149 ~TestBuffer() override {
150 if (event_)
151 event_->Set();
152 }
153 rtc::Event* const event_;
154};
155
Henrik Boström56db9ff2021-03-24 09:06:45 +0100156// A fake native buffer that can't be converted to I420. Upon scaling, it
157// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700158class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
159 public:
160 FakeNativeBuffer(rtc::Event* event, int width, int height)
161 : event_(event), width_(width), height_(height) {}
162 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
163 int width() const override { return width_; }
164 int height() const override { return height_; }
165 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
166 return nullptr;
167 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100168 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
169 int offset_x,
170 int offset_y,
171 int crop_width,
172 int crop_height,
173 int scaled_width,
174 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200175 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
176 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100177 }
Noah Richards51db4212019-06-12 06:59:12 -0700178
179 private:
180 friend class rtc::RefCountedObject<FakeNativeBuffer>;
181 ~FakeNativeBuffer() override {
182 if (event_)
183 event_->Set();
184 }
185 rtc::Event* const event_;
186 const int width_;
187 const int height_;
188};
189
Evan Shrubsole895556e2020-10-05 09:15:13 +0200190// A fake native buffer that is backed by an NV12 buffer.
191class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
192 public:
193 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
194 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
195
196 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
197 int width() const override { return nv12_buffer_->width(); }
198 int height() const override { return nv12_buffer_->height(); }
199 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
200 return nv12_buffer_->ToI420();
201 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200202 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
203 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
204 if (absl::c_find(types, Type::kNV12) != types.end()) {
205 return nv12_buffer_;
206 }
207 return nullptr;
208 }
Niels Möllerba2de582022-04-20 16:46:26 +0200209 const NV12BufferInterface* GetNV12() const { return nv12_buffer_.get(); }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200210
211 private:
212 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
213 ~FakeNV12NativeBuffer() override {
214 if (event_)
215 event_->Set();
216 }
217 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
218 rtc::Event* const event_;
219};
220
Niels Möller7dc26b72017-12-06 10:27:48 +0100221class CpuOveruseDetectorProxy : public OveruseFrameDetector {
222 public:
Markus Handell8e4197b2022-05-30 15:45:28 +0200223 CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer,
224 const FieldTrialsView& field_trials)
225 : OveruseFrameDetector(metrics_observer, field_trials),
Henrik Boström381d1092020-05-12 18:49:07 +0200226 last_target_framerate_fps_(-1),
227 framerate_updated_event_(true /* manual_reset */,
228 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100229 virtual ~CpuOveruseDetectorProxy() {}
230
231 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200232 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100233 last_target_framerate_fps_ = framerate_fps;
234 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200235 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100236 }
237
238 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200239 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100240 return last_target_framerate_fps_;
241 }
242
Niels Möller4db138e2018-04-19 09:04:13 +0200243 CpuOveruseOptions GetOptions() { return options_; }
244
Henrik Boström381d1092020-05-12 18:49:07 +0200245 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
246
Niels Möller7dc26b72017-12-06 10:27:48 +0100247 private:
Markus Handella3765182020-07-08 13:13:32 +0200248 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100249 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200250 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100251};
252
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200253class FakeVideoSourceRestrictionsListener
254 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200255 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200256 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200257 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200258 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200259 RTC_DCHECK(was_restrictions_updated_);
260 }
261
262 rtc::Event* restrictions_updated_event() {
263 return &restrictions_updated_event_;
264 }
265
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200266 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200267 void OnVideoSourceRestrictionsUpdated(
268 VideoSourceRestrictions restrictions,
269 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200270 rtc::scoped_refptr<Resource> reason,
271 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200272 was_restrictions_updated_ = true;
273 restrictions_updated_event_.Set();
274 }
275
276 private:
277 bool was_restrictions_updated_;
278 rtc::Event restrictions_updated_event_;
279};
280
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200281auto WantsFps(Matcher<int> fps_matcher) {
282 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
283 fps_matcher);
284}
285
286auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
287 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
288 AllOf(max_pixel_matcher, Gt(0)));
289}
290
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200291auto ResolutionMax() {
292 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200293 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200294 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
295 Eq(absl::nullopt)));
296}
297
298auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200299 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200300}
301
302auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200303 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200304}
305
306auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200307 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200308}
309
310auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200311 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200312}
313
314auto FpsMaxResolutionMax() {
315 return AllOf(FpsMax(), ResolutionMax());
316}
317
318auto UnlimitedSinkWants() {
319 return AllOf(FpsUnlimited(), ResolutionMax());
320}
321
322auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
323 Matcher<int> fps_range_matcher;
324
325 if (last_frame_pixels <= 320 * 240) {
326 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200327 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200328 fps_range_matcher = AllOf(Ge(10), Le(15));
329 } else if (last_frame_pixels <= 640 * 480) {
330 fps_range_matcher = Ge(15);
331 } else {
332 fps_range_matcher = Eq(kDefaultFramerate);
333 }
334 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
335 fps_range_matcher);
336}
337
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200338auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
339 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
340 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
341}
342
343auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
344 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
345}
346
347auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
348 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
349}
350
351auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
352 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
353 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
354}
355
356auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
357 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
358 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
359}
360
361auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
362 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
363 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
364}
365
366auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
367 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
368 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
369}
370
mflodmancc3d4422017-08-03 08:27:51 -0700371class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700372 public:
Markus Handell9a478b52021-11-18 16:07:01 +0100373 VideoStreamEncoderUnderTest(
374 TimeController* time_controller,
375 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter,
376 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
377 encoder_queue,
378 SendStatisticsProxy* stats_proxy,
379 const VideoStreamEncoderSettings& settings,
380 VideoStreamEncoder::BitrateAllocationCallbackType
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100381 allocation_callback_type,
Erik Språnge4589cb2022-04-06 16:44:30 +0200382 const FieldTrialsView& field_trials,
383 int num_cores)
Markus Handell8e4197b2022-05-30 15:45:28 +0200384 : VideoStreamEncoder(
385 time_controller->GetClock(),
386 num_cores,
387 stats_proxy,
388 settings,
389 std::unique_ptr<OveruseFrameDetector>(
390 overuse_detector_proxy_ =
391 new CpuOveruseDetectorProxy(stats_proxy, field_trials)),
392 std::move(cadence_adapter),
393 std::move(encoder_queue),
394 allocation_callback_type,
395 field_trials),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200396 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200397 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200398 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200399 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200400 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200401 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200402 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200403 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100404 }
perkj803d97f2016-11-01 11:45:46 -0700405
Henrik Boström381d1092020-05-12 18:49:07 +0200406 void SetSourceAndWaitForRestrictionsUpdated(
407 rtc::VideoSourceInterface<VideoFrame>* source,
408 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200409 FakeVideoSourceRestrictionsListener listener;
410 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200411 SetSource(source, degradation_preference);
412 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200413 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200414 }
415
416 void SetSourceAndWaitForFramerateUpdated(
417 rtc::VideoSourceInterface<VideoFrame>* source,
418 const DegradationPreference& degradation_preference) {
419 overuse_detector_proxy_->framerate_updated_event()->Reset();
420 SetSource(source, degradation_preference);
421 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
422 }
423
424 void OnBitrateUpdatedAndWaitForManagedResources(
425 DataRate target_bitrate,
426 DataRate stable_target_bitrate,
427 DataRate link_allocation,
428 uint8_t fraction_lost,
429 int64_t round_trip_time_ms,
430 double cwnd_reduce_ratio) {
431 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
432 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
433 // Bitrate is updated on the encoder queue.
434 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200435 }
436
kthelgason2fc52542017-03-03 00:24:41 -0800437 // This is used as a synchronisation mechanism, to make sure that the
438 // encoder queue is not blocked before we start sending it frames.
439 void WaitUntilTaskQueueIsIdle() {
Markus Handell28c71802021-11-08 10:11:55 +0100440 time_controller_->AdvanceTime(TimeDelta::Zero());
kthelgason2fc52542017-03-03 00:24:41 -0800441 }
442
Henrik Boström91aa7322020-04-28 12:24:33 +0200443 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200444 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200445 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200446 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200447 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200448 event.Set();
449 });
450 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100451 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200452 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200453
Henrik Boström91aa7322020-04-28 12:24:33 +0200454 void TriggerCpuUnderuse() {
455 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200456 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200457 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200458 event.Set();
459 });
460 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100461 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200462 }
kthelgason876222f2016-11-29 01:44:11 -0800463
Henrik Boström91aa7322020-04-28 12:24:33 +0200464 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200465 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200466 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200467 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200468 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200469 event.Set();
470 });
471 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100472 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200473 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200474 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200475 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200476 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200477 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200478 event.Set();
479 });
480 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100481 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200482 }
483
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200484 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100485 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200486 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
487 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200488 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700489};
490
Noah Richards51db4212019-06-12 06:59:12 -0700491// Simulates simulcast behavior and makes highest stream resolutions divisible
492// by 4.
493class CroppingVideoStreamFactory
494 : public VideoEncoderConfig::VideoStreamFactoryInterface {
495 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200496 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700497
498 private:
499 std::vector<VideoStream> CreateEncoderStreams(
500 int width,
501 int height,
502 const VideoEncoderConfig& encoder_config) override {
503 std::vector<VideoStream> streams = test::CreateVideoStreams(
504 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700505 return streams;
506 }
Noah Richards51db4212019-06-12 06:59:12 -0700507};
508
sprangb1ca0732017-02-01 08:38:12 -0800509class AdaptingFrameForwarder : public test::FrameForwarder {
510 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200511 explicit AdaptingFrameForwarder(TimeController* time_controller)
512 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700513 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800514
515 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200516 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800517 adaptation_enabled_ = enabled;
518 }
519
asaperssonfab67072017-04-04 05:51:49 -0700520 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200521 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800522 return adaptation_enabled_;
523 }
524
Henrik Boström1124ed12021-02-25 10:30:39 +0100525 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
526 // the resolution or frame rate was different than it is currently. If
527 // something else is modified, such as encoder resolutions, but the resolution
528 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700529 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200530 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700531 return last_wants_;
532 }
533
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200534 absl::optional<int> last_sent_width() const { return last_width_; }
535 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800536
sprangb1ca0732017-02-01 08:38:12 -0800537 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200538 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +0100539 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200540
sprangb1ca0732017-02-01 08:38:12 -0800541 int cropped_width = 0;
542 int cropped_height = 0;
543 int out_width = 0;
544 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700545 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000546 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
547 << "w=" << video_frame.width()
548 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700549 if (adapter_.AdaptFrameResolution(
550 video_frame.width(), video_frame.height(),
551 video_frame.timestamp_us() * 1000, &cropped_width,
552 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100553 VideoFrame adapted_frame =
554 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200555 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100556 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200557 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100558 .set_timestamp_ms(99)
559 .set_rotation(kVideoRotation_0)
560 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100561 if (video_frame.has_update_rect()) {
562 adapted_frame.set_update_rect(
563 video_frame.update_rect().ScaleWithFrame(
564 video_frame.width(), video_frame.height(), 0, 0,
565 video_frame.width(), video_frame.height(), out_width,
566 out_height));
567 }
sprangc5d62e22017-04-02 23:53:04 -0700568 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800569 last_width_.emplace(adapted_frame.width());
570 last_height_.emplace(adapted_frame.height());
571 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200572 last_width_ = absl::nullopt;
573 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700574 }
sprangb1ca0732017-02-01 08:38:12 -0800575 } else {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000576 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800577 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800578 last_width_.emplace(video_frame.width());
579 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800580 }
581 }
582
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200583 void OnOutputFormatRequest(int width, int height) {
584 absl::optional<std::pair<int, int>> target_aspect_ratio =
585 std::make_pair(width, height);
586 absl::optional<int> max_pixel_count = width * height;
587 absl::optional<int> max_fps;
588 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
589 max_fps);
590 }
591
sprangb1ca0732017-02-01 08:38:12 -0800592 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
593 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200594 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100595 rtc::VideoSinkWants prev_wants = sink_wants_locked();
596 bool did_adapt =
597 prev_wants.max_pixel_count != wants.max_pixel_count ||
598 prev_wants.target_pixel_count != wants.target_pixel_count ||
599 prev_wants.max_framerate_fps != wants.max_framerate_fps;
600 if (did_adapt) {
601 last_wants_ = prev_wants;
602 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100603 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200604 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800605 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200606
Erik Språng5e13d052022-08-02 11:42:49 +0200607 void RequestRefreshFrame() override { ++refresh_frames_requested_; }
608
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200609 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800610 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200611 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
612 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200613 absl::optional<int> last_width_;
614 absl::optional<int> last_height_;
Erik Språng5e13d052022-08-02 11:42:49 +0200615 int refresh_frames_requested_{0};
sprangb1ca0732017-02-01 08:38:12 -0800616};
sprangc5d62e22017-04-02 23:53:04 -0700617
Niels Möller213618e2018-07-24 09:29:58 +0200618// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700619class MockableSendStatisticsProxy : public SendStatisticsProxy {
620 public:
621 MockableSendStatisticsProxy(Clock* clock,
622 const VideoSendStream::Config& config,
Jonas Oreland8ca06132022-03-14 12:52:48 +0100623 VideoEncoderConfig::ContentType content_type,
Jonas Orelande62c2f22022-03-29 11:04:48 +0200624 const FieldTrialsView& field_trials)
Jonas Oreland8ca06132022-03-14 12:52:48 +0100625 : SendStatisticsProxy(clock, config, content_type, field_trials) {}
sprangc5d62e22017-04-02 23:53:04 -0700626
627 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200628 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700629 if (mock_stats_)
630 return *mock_stats_;
631 return SendStatisticsProxy::GetStats();
632 }
633
Niels Möller213618e2018-07-24 09:29:58 +0200634 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200635 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200636 if (mock_stats_)
637 return mock_stats_->input_frame_rate;
638 return SendStatisticsProxy::GetInputFrameRate();
639 }
sprangc5d62e22017-04-02 23:53:04 -0700640 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200641 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700642 mock_stats_.emplace(stats);
643 }
644
645 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200646 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700647 mock_stats_.reset();
648 }
649
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200650 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
651 on_frame_dropped_ = std::move(callback);
652 }
653
sprangc5d62e22017-04-02 23:53:04 -0700654 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200655 void OnFrameDropped(DropReason reason) override {
656 SendStatisticsProxy::OnFrameDropped(reason);
657 if (on_frame_dropped_)
658 on_frame_dropped_(reason);
659 }
660
Markus Handella3765182020-07-08 13:13:32 +0200661 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200662 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200663 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700664};
665
Markus Handellb4e96d42021-11-05 12:00:55 +0100666class SimpleVideoStreamEncoderFactory {
667 public:
668 class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
669 public:
670 using VideoStreamEncoder::VideoStreamEncoder;
671 ~AdaptedVideoStreamEncoder() { Stop(); }
672 };
673
Markus Handell8d87c462021-12-16 11:37:16 +0100674 class MockFakeEncoder : public test::FakeEncoder {
675 public:
676 using FakeEncoder::FakeEncoder;
677 MOCK_METHOD(CodecSpecificInfo,
678 EncodeHook,
679 (EncodedImage & encoded_image,
680 rtc::scoped_refptr<EncodedImageBuffer> buffer),
681 (override));
682 };
683
Markus Handellee225432021-11-29 12:35:12 +0100684 SimpleVideoStreamEncoderFactory() {
Markus Handellb4e96d42021-11-05 12:00:55 +0100685 encoder_settings_.encoder_factory = &encoder_factory_;
Markus Handellee225432021-11-29 12:35:12 +0100686 encoder_settings_.bitrate_allocator_factory =
687 bitrate_allocator_factory_.get();
Markus Handellb4e96d42021-11-05 12:00:55 +0100688 }
689
Markus Handell818e7fb2021-12-30 13:01:33 +0100690 std::unique_ptr<AdaptedVideoStreamEncoder> CreateWithEncoderQueue(
Markus Handellee225432021-11-29 12:35:12 +0100691 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100692 std::unique_ptr<TaskQueueBase, TaskQueueDeleter> encoder_queue,
Jonas Orelande62c2f22022-03-29 11:04:48 +0200693 const FieldTrialsView* field_trials = nullptr) {
Markus Handellb4e96d42021-11-05 12:00:55 +0100694 auto result = std::make_unique<AdaptedVideoStreamEncoder>(
695 time_controller_.GetClock(),
696 /*number_of_cores=*/1,
697 /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
Markus Handell8e4197b2022-05-30 15:45:28 +0200698 std::make_unique<CpuOveruseDetectorProxy>(
699 /*stats_proxy=*/nullptr,
700 field_trials ? *field_trials : field_trials_),
Markus Handellee225432021-11-29 12:35:12 +0100701 std::move(zero_hertz_adapter), std::move(encoder_queue),
Markus Handellb4e96d42021-11-05 12:00:55 +0100702 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100703 kVideoBitrateAllocation,
704 field_trials ? *field_trials : field_trials_);
Markus Handellb4e96d42021-11-05 12:00:55 +0100705 result->SetSink(&sink_, /*rotation_applied=*/false);
706 return result;
707 }
708
Markus Handell818e7fb2021-12-30 13:01:33 +0100709 std::unique_ptr<AdaptedVideoStreamEncoder> Create(
710 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
711 TaskQueueBase** encoder_queue_ptr = nullptr) {
712 auto encoder_queue =
713 time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
714 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
715 if (encoder_queue_ptr)
716 *encoder_queue_ptr = encoder_queue.get();
717 return CreateWithEncoderQueue(std::move(zero_hertz_adapter),
718 std::move(encoder_queue));
719 }
720
Markus Handell9a478b52021-11-18 16:07:01 +0100721 void DepleteTaskQueues() { time_controller_.AdvanceTime(TimeDelta::Zero()); }
Markus Handell8d87c462021-12-16 11:37:16 +0100722 MockFakeEncoder& GetMockFakeEncoder() { return mock_fake_encoder_; }
Markus Handell9a478b52021-11-18 16:07:01 +0100723
Markus Handell818e7fb2021-12-30 13:01:33 +0100724 GlobalSimulatedTimeController* GetTimeController() {
725 return &time_controller_;
726 }
727
Markus Handellb4e96d42021-11-05 12:00:55 +0100728 private:
729 class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
730 public:
731 ~NullEncoderSink() override = default;
732 void OnEncoderConfigurationChanged(
733 std::vector<VideoStream> streams,
734 bool is_svc,
735 VideoEncoderConfig::ContentType content_type,
736 int min_transmit_bitrate_bps) override {}
737 void OnBitrateAllocationUpdated(
738 const VideoBitrateAllocation& allocation) override {}
739 void OnVideoLayersAllocationUpdated(
740 VideoLayersAllocation allocation) override {}
741 Result OnEncodedImage(
742 const EncodedImage& encoded_image,
743 const CodecSpecificInfo* codec_specific_info) override {
744 return Result(EncodedImageCallback::Result::OK);
745 }
746 };
747
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100748 test::ScopedKeyValueConfig field_trials_;
Markus Handellee225432021-11-29 12:35:12 +0100749 GlobalSimulatedTimeController time_controller_{Timestamp::Millis(0)};
750 std::unique_ptr<TaskQueueFactory> task_queue_factory_{
751 time_controller_.CreateTaskQueueFactory()};
752 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_ =
753 std::make_unique<MockableSendStatisticsProxy>(
754 time_controller_.GetClock(),
755 VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 12:52:48 +0100756 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
757 field_trials_);
Markus Handellee225432021-11-29 12:35:12 +0100758 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_ =
759 CreateBuiltinVideoBitrateAllocatorFactory();
760 VideoStreamEncoderSettings encoder_settings_{
761 VideoEncoder::Capabilities(/*loss_notification=*/false)};
Markus Handell8d87c462021-12-16 11:37:16 +0100762 MockFakeEncoder mock_fake_encoder_{time_controller_.GetClock()};
763 test::VideoEncoderProxyFactory encoder_factory_{&mock_fake_encoder_};
Markus Handellb4e96d42021-11-05 12:00:55 +0100764 NullEncoderSink sink_;
765};
766
767class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
768 public:
769 MOCK_METHOD(void, Initialize, (Callback * callback), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100770 MOCK_METHOD(void,
771 SetZeroHertzModeEnabled,
772 (absl::optional<ZeroHertzModeParams>),
773 (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100774 MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
Markus Handellee225432021-11-29 12:35:12 +0100775 MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override));
776 MOCK_METHOD(void, UpdateFrameRate, (), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100777 MOCK_METHOD(void,
778 UpdateLayerQualityConvergence,
779 (int spatial_index, bool converged),
780 (override));
781 MOCK_METHOD(void,
782 UpdateLayerStatus,
783 (int spatial_index, bool enabled),
784 (override));
Markus Handell818e7fb2021-12-30 13:01:33 +0100785 MOCK_METHOD(void, ProcessKeyFrameRequest, (), (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100786};
787
philipel9b058032020-02-10 11:30:00 +0100788class MockEncoderSelector
789 : public VideoEncoderFactory::EncoderSelectorInterface {
790 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200791 MOCK_METHOD(void,
792 OnCurrentEncoder,
793 (const SdpVideoFormat& format),
794 (override));
795 MOCK_METHOD(absl::optional<SdpVideoFormat>,
796 OnAvailableBitrate,
797 (const DataRate& rate),
798 (override));
philipel6daa3042022-04-11 10:48:28 +0200799 MOCK_METHOD(absl::optional<SdpVideoFormat>,
800 OnResolutionChange,
801 (const RenderResolution& resolution),
802 (override));
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200803 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100804};
805
Markus Handell2e0f4f02021-12-21 19:14:58 +0100806class MockVideoSourceInterface : public rtc::VideoSourceInterface<VideoFrame> {
807 public:
808 MOCK_METHOD(void,
809 AddOrUpdateSink,
810 (rtc::VideoSinkInterface<VideoFrame>*,
811 const rtc::VideoSinkWants&),
812 (override));
813 MOCK_METHOD(void,
814 RemoveSink,
815 (rtc::VideoSinkInterface<VideoFrame>*),
816 (override));
817 MOCK_METHOD(void, RequestRefreshFrame, (), (override));
818};
819
perkj803d97f2016-11-01 11:45:46 -0700820} // namespace
821
mflodmancc3d4422017-08-03 08:27:51 -0700822class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700823 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200824 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700825
mflodmancc3d4422017-08-03 08:27:51 -0700826 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700827 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700828 codec_width_(320),
829 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200830 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200831 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200832 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700833 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200834 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700835 video_send_config_,
Jonas Oreland8ca06132022-03-14 12:52:48 +0100836 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
837 field_trials_)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200838 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700839
840 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700841 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700842 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200843 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800844 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200845 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200846 video_send_config_.rtp.payload_name = "FAKE";
847 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700848
Per512ecb32016-09-23 15:52:06 +0200849 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200850 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200851 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
852 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
853 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100854 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700855
Niels Möllerf1338562018-04-26 09:51:47 +0200856 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800857 }
858
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100859 void ConfigureEncoder(
860 VideoEncoderConfig video_encoder_config,
861 VideoStreamEncoder::BitrateAllocationCallbackType
862 allocation_callback_type =
863 VideoStreamEncoder::BitrateAllocationCallbackType::
Erik Språnge4589cb2022-04-06 16:44:30 +0200864 kVideoBitrateAllocationWhenScreenSharing,
865 int num_cores = 1) {
mflodmancc3d4422017-08-03 08:27:51 -0700866 if (video_stream_encoder_)
867 video_stream_encoder_->Stop();
Markus Handell9a478b52021-11-18 16:07:01 +0100868
869 auto encoder_queue = GetTaskQueueFactory()->CreateTaskQueue(
870 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
871 TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
872 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter =
873 FrameCadenceAdapterInterface::Create(time_controller_.GetClock(),
Jonas Oreland8ca06132022-03-14 12:52:48 +0100874 encoder_queue_ptr, field_trials_);
Markus Handell9a478b52021-11-18 16:07:01 +0100875 video_stream_encoder_ = std::make_unique<VideoStreamEncoderUnderTest>(
876 &time_controller_, std::move(cadence_adapter), std::move(encoder_queue),
877 stats_proxy_.get(), video_send_config_.encoder_settings,
Erik Språnge4589cb2022-04-06 16:44:30 +0200878 allocation_callback_type, field_trials_, num_cores);
Asa Persson606d3cb2021-10-04 10:07:11 +0200879 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 08:27:51 -0700880 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700881 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 10:07:11 +0200882 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 08:27:51 -0700883 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200884 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700885 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800886 }
887
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100888 void ResetEncoder(const std::string& payload_name,
889 size_t num_streams,
890 size_t num_temporal_layers,
891 unsigned char num_spatial_layers,
892 bool screenshare,
893 VideoStreamEncoder::BitrateAllocationCallbackType
894 allocation_callback_type =
895 VideoStreamEncoder::BitrateAllocationCallbackType::
Erik Språnge4589cb2022-04-06 16:44:30 +0200896 kVideoBitrateAllocationWhenScreenSharing,
897 int num_cores = 1) {
Niels Möller259a4972018-04-05 15:36:51 +0200898 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800899
900 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200901 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
902 num_streams, &video_encoder_config);
903 for (auto& layer : video_encoder_config.simulcast_layers) {
904 layer.num_temporal_layers = num_temporal_layers;
905 layer.max_framerate = kDefaultFramerate;
906 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100907 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 10:07:11 +0200908 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 07:06:52 -0700909 video_encoder_config.content_type =
910 screenshare ? VideoEncoderConfig::ContentType::kScreen
911 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700912 if (payload_name == "VP9") {
913 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
914 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200915 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700916 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200917 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
918 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700919 }
Erik Språnge4589cb2022-04-06 16:44:30 +0200920 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type,
921 num_cores);
perkj26091b12016-09-01 01:17:40 -0700922 }
923
sprang57c2fff2017-01-16 06:24:02 -0800924 VideoFrame CreateFrame(int64_t ntp_time_ms,
925 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200926 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200927 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200928 destruction_event, codec_width_, codec_height_))
929 .set_ntp_time_ms(ntp_time_ms)
930 .set_timestamp_ms(99)
931 .set_rotation(kVideoRotation_0)
932 .build();
perkj26091b12016-09-01 01:17:40 -0700933 }
934
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100935 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
936 rtc::Event* destruction_event,
937 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200938 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200939 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200940 destruction_event, codec_width_, codec_height_))
941 .set_ntp_time_ms(ntp_time_ms)
942 .set_timestamp_ms(99)
943 .set_rotation(kVideoRotation_0)
944 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
945 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100946 }
947
sprang57c2fff2017-01-16 06:24:02 -0800948 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200949 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
950 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200951 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200952 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200953 .set_ntp_time_ms(ntp_time_ms)
954 .set_timestamp_ms(ntp_time_ms)
955 .set_rotation(kVideoRotation_0)
956 .build();
perkj803d97f2016-11-01 11:45:46 -0700957 }
958
Evan Shrubsole895556e2020-10-05 09:15:13 +0200959 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200960 return VideoFrame::Builder()
961 .set_video_frame_buffer(NV12Buffer::Create(width, height))
962 .set_ntp_time_ms(ntp_time_ms)
963 .set_timestamp_ms(ntp_time_ms)
964 .set_rotation(kVideoRotation_0)
965 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200966 }
967
Noah Richards51db4212019-06-12 06:59:12 -0700968 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
969 rtc::Event* destruction_event,
970 int width,
971 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200972 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200973 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200974 destruction_event, width, height))
975 .set_ntp_time_ms(ntp_time_ms)
976 .set_timestamp_ms(99)
977 .set_rotation(kVideoRotation_0)
978 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700979 }
980
Evan Shrubsole895556e2020-10-05 09:15:13 +0200981 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
982 rtc::Event* destruction_event,
983 int width,
984 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200985 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200986 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200987 destruction_event, width, height))
988 .set_ntp_time_ms(ntp_time_ms)
989 .set_timestamp_ms(99)
990 .set_rotation(kVideoRotation_0)
991 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200992 }
993
Noah Richards51db4212019-06-12 06:59:12 -0700994 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
995 rtc::Event* destruction_event) const {
996 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
997 codec_height_);
998 }
999
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001000 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001001 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001002 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001003
1004 video_source_.IncomingCapturedFrame(
1005 CreateFrame(1, codec_width_, codec_height_));
1006 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +02001007 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001008 }
1009
sprang4847ae62017-06-27 07:06:52 -07001010 void WaitForEncodedFrame(int64_t expected_ntp_time) {
1011 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001012 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001013 }
1014
1015 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
1016 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001017 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001018 return ok;
1019 }
1020
1021 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
1022 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001023 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001024 }
1025
1026 void ExpectDroppedFrame() {
1027 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001028 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001029 }
1030
1031 bool WaitForFrame(int64_t timeout_ms) {
1032 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001033 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001034 return ok;
1035 }
1036
perkj26091b12016-09-01 01:17:40 -07001037 class TestEncoder : public test::FakeEncoder {
1038 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001039 explicit TestEncoder(TimeController* time_controller)
1040 : FakeEncoder(time_controller->GetClock()),
1041 time_controller_(time_controller) {
1042 RTC_DCHECK(time_controller_);
1043 }
perkj26091b12016-09-01 01:17:40 -07001044
Erik Språngaed30702018-11-05 12:57:17 +01001045 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +02001046 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +02001047 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +01001048 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001049 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +01001050 info.scaling_settings = VideoEncoder::ScalingSettings(
1051 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001052 }
1053 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001054 for (int i = 0; i < kMaxSpatialLayers; ++i) {
1055 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001056 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001057 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001058 for (int tid = 0; tid < num_layers; ++tid)
1059 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001060 }
1061 }
Erik Språngaed30702018-11-05 12:57:17 +01001062 }
Sergey Silkin6456e352019-07-08 17:56:40 +02001063
1064 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001065 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001066 info.apply_alignment_to_all_simulcast_layers =
1067 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001068 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001069 if (is_qp_trusted_.has_value()) {
1070 info.is_qp_trusted = is_qp_trusted_;
1071 }
Erik Språngaed30702018-11-05 12:57:17 +01001072 return info;
kthelgason876222f2016-11-29 01:44:11 -08001073 }
1074
Erik Språngb7cb7b52019-02-26 15:52:33 +01001075 int32_t RegisterEncodeCompleteCallback(
1076 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +02001077 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001078 encoded_image_callback_ = callback;
1079 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
1080 }
1081
perkjfa10b552016-10-02 23:45:26 -07001082 void ContinueEncode() { continue_encode_event_.Set(); }
1083
1084 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
1085 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +02001086 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -07001087 EXPECT_EQ(timestamp_, timestamp);
1088 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
1089 }
1090
kthelgason2fc52542017-03-03 00:24:41 -08001091 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +02001092 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -08001093 quality_scaling_ = b;
1094 }
kthelgasonad9010c2017-02-14 00:46:51 -08001095
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001096 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +02001097 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001098 requested_resolution_alignment_ = requested_resolution_alignment;
1099 }
1100
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001101 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
1102 MutexLock lock(&local_mutex_);
1103 apply_alignment_to_all_simulcast_layers_ = b;
1104 }
1105
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001106 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +02001107 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001108 is_hardware_accelerated_ = is_hardware_accelerated;
1109 }
1110
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001111 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
1112 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +02001113 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001114 temporal_layers_supported_[spatial_idx] = supported;
1115 }
1116
Sergey Silkin6456e352019-07-08 17:56:40 +02001117 void SetResolutionBitrateLimits(
1118 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +02001119 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +02001120 resolution_bitrate_limits_ = thresholds;
1121 }
1122
sprangfe627f32017-03-29 08:24:59 -07001123 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +02001124 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -07001125 force_init_encode_failed_ = force_failure;
1126 }
1127
Niels Möller6bb5ab92019-01-11 11:11:10 +01001128 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +02001129 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001130 rate_factor_ = rate_factor;
1131 }
1132
Erik Språngd7329ca2019-02-21 21:19:53 +01001133 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +02001134 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001135 return last_framerate_;
1136 }
1137
Erik Språngd7329ca2019-02-21 21:19:53 +01001138 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +02001139 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001140 return last_update_rect_;
1141 }
1142
Niels Möller87e2d782019-03-07 10:18:23 +01001143 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +02001144 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001145 return last_frame_types_;
1146 }
1147
1148 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +01001149 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +01001150 keyframe ? VideoFrameType::kVideoFrameKey
1151 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01001152 {
Markus Handella3765182020-07-08 13:13:32 +02001153 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001154 last_frame_types_ = frame_type;
1155 }
Niels Möllerb859b322019-03-07 12:40:01 +01001156 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +01001157 }
1158
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001159 void InjectEncodedImage(const EncodedImage& image,
1160 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +02001161 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001162 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001163 }
1164
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001165 void SetEncodedImageData(
1166 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +02001167 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001168 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001169 }
1170
Erik Språngd7329ca2019-02-21 21:19:53 +01001171 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +02001172 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001173 expect_null_frame_ = true;
1174 }
1175
Erik Språng5056af02019-09-02 15:53:11 +02001176 absl::optional<VideoEncoder::RateControlParameters>
1177 GetAndResetLastRateControlSettings() {
1178 auto settings = last_rate_control_settings_;
1179 last_rate_control_settings_.reset();
1180 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +01001181 }
1182
Henrik Boström56db9ff2021-03-24 09:06:45 +01001183 int GetLastInputWidth() const {
1184 MutexLock lock(&local_mutex_);
1185 return last_input_width_;
1186 }
1187
1188 int GetLastInputHeight() const {
1189 MutexLock lock(&local_mutex_);
1190 return last_input_height_;
1191 }
1192
Evan Shrubsole895556e2020-10-05 09:15:13 +02001193 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1194 MutexLock lock(&local_mutex_);
1195 return last_input_pixel_format_;
1196 }
1197
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001198 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001199 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001200 return num_set_rates_;
1201 }
1202
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001203 void SetPreferredPixelFormats(
1204 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1205 pixel_formats) {
1206 MutexLock lock(&local_mutex_);
1207 preferred_pixel_formats_ = std::move(pixel_formats);
1208 }
1209
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001210 void SetIsQpTrusted(absl::optional<bool> trusted) {
1211 MutexLock lock(&local_mutex_);
1212 is_qp_trusted_ = trusted;
1213 }
1214
Erik Språnge4589cb2022-04-06 16:44:30 +02001215 VideoCodecComplexity LastEncoderComplexity() {
1216 MutexLock lock(&local_mutex_);
1217 return last_encoder_complexity_;
1218 }
1219
perkjfa10b552016-10-02 23:45:26 -07001220 private:
perkj26091b12016-09-01 01:17:40 -07001221 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001222 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001223 {
Markus Handella3765182020-07-08 13:13:32 +02001224 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001225 if (expect_null_frame_) {
1226 EXPECT_EQ(input_image.timestamp(), 0u);
1227 EXPECT_EQ(input_image.width(), 1);
1228 last_frame_types_ = *frame_types;
1229 expect_null_frame_ = false;
1230 } else {
1231 EXPECT_GT(input_image.timestamp(), timestamp_);
1232 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1233 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1234 }
perkj26091b12016-09-01 01:17:40 -07001235
1236 timestamp_ = input_image.timestamp();
1237 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001238 last_input_width_ = input_image.width();
1239 last_input_height_ = input_image.height();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001240 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001241 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001242 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001243 }
Niels Möllerb859b322019-03-07 12:40:01 +01001244 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001245 return result;
1246 }
1247
Niels Möller08ae7ce2020-09-23 15:58:12 +02001248 CodecSpecificInfo EncodeHook(
1249 EncodedImage& encoded_image,
1250 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001251 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001252 {
1253 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001254 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001255 }
1256 MutexLock lock(&local_mutex_);
1257 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001258 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001259 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001260 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001261 }
1262
sprangfe627f32017-03-29 08:24:59 -07001263 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001264 const Settings& settings) override {
1265 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001266
Markus Handella3765182020-07-08 13:13:32 +02001267 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001268 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001269
Erik Språng82fad3d2018-03-21 09:57:23 +01001270 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001271 // Simulate setting up temporal layers, in order to validate the life
1272 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001273 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001274 frame_buffer_controller_ =
1275 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001276 }
Erik Språnge4589cb2022-04-06 16:44:30 +02001277
1278 last_encoder_complexity_ = config->GetVideoEncoderComplexity();
1279
Erik Språngb7cb7b52019-02-26 15:52:33 +01001280 if (force_init_encode_failed_) {
1281 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001282 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001283 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001284
Erik Språngb7cb7b52019-02-26 15:52:33 +01001285 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001286 return res;
1287 }
1288
Erik Språngb7cb7b52019-02-26 15:52:33 +01001289 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001290 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001291 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1292 initialized_ = EncoderState::kUninitialized;
1293 return FakeEncoder::Release();
1294 }
1295
Erik Språng16cb8f52019-04-12 13:59:09 +02001296 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001297 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001298 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001299 VideoBitrateAllocation adjusted_rate_allocation;
1300 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1301 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001302 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001303 adjusted_rate_allocation.SetBitrate(
1304 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001305 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001306 rate_factor_));
1307 }
1308 }
1309 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001310 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001311 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001312 RateControlParameters adjusted_paramters = parameters;
1313 adjusted_paramters.bitrate = adjusted_rate_allocation;
1314 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001315 }
1316
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001317 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001318 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001319 enum class EncoderState {
1320 kUninitialized,
1321 kInitializationFailed,
1322 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001323 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
perkj26091b12016-09-01 01:17:40 -07001324 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001325 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1326 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1327 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1328 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1329 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1330 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001331 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1332 false;
Markus Handella3765182020-07-08 13:13:32 +02001333 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001334 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1335 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001336 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001337 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001338 absl::optional<bool>
1339 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001340 local_mutex_);
1341 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1342 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1343 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001344 absl::optional<VideoEncoder::RateControlParameters>
1345 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001346 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1347 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001348 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001349 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001350 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1351 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001352 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 17:56:40 +02001353 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001354 RTC_GUARDED_BY(local_mutex_);
1355 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001356 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1357 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001358 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1359 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001360 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
Erik Språnge4589cb2022-04-06 16:44:30 +02001361 VideoCodecComplexity last_encoder_complexity_ RTC_GUARDED_BY(local_mutex_){
1362 VideoCodecComplexity::kComplexityNormal};
perkj26091b12016-09-01 01:17:40 -07001363 };
1364
mflodmancc3d4422017-08-03 08:27:51 -07001365 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001366 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001367 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1368 : time_controller_(time_controller), test_encoder_(test_encoder) {
1369 RTC_DCHECK(time_controller_);
1370 }
perkj26091b12016-09-01 01:17:40 -07001371
perkj26091b12016-09-01 01:17:40 -07001372 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001373 EXPECT_TRUE(
1374 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1375 }
1376
1377 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1378 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001379 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001380 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001381 return false;
perkj26091b12016-09-01 01:17:40 -07001382 {
Markus Handella3765182020-07-08 13:13:32 +02001383 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001384 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001385 }
1386 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001387 return true;
perkj26091b12016-09-01 01:17:40 -07001388 }
1389
sprangb1ca0732017-02-01 08:38:12 -08001390 void WaitForEncodedFrame(uint32_t expected_width,
1391 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001392 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001393 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001394 }
1395
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001396 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001397 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001398 uint32_t width = 0;
1399 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001400 {
Markus Handella3765182020-07-08 13:13:32 +02001401 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001402 width = last_width_;
1403 height = last_height_;
1404 }
1405 EXPECT_EQ(expected_height, height);
1406 EXPECT_EQ(expected_width, width);
1407 }
1408
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001409 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1410 VideoRotation rotation;
1411 {
Markus Handella3765182020-07-08 13:13:32 +02001412 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001413 rotation = last_rotation_;
1414 }
1415 EXPECT_EQ(expected_rotation, rotation);
1416 }
1417
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001418 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001419
sprangc5d62e22017-04-02 23:53:04 -07001420 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001421 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +01001422 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001423 bool ret = encoded_frame_event_.Wait(timeout_ms);
Markus Handell28c71802021-11-08 10:11:55 +01001424 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001425 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001426 }
1427
perkj26091b12016-09-01 01:17:40 -07001428 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001429 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001430 expect_frames_ = false;
1431 }
1432
asaperssonfab67072017-04-04 05:51:49 -07001433 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001434 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001435 return number_of_reconfigurations_;
1436 }
1437
asaperssonfab67072017-04-04 05:51:49 -07001438 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001439 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001440 return min_transmit_bitrate_bps_;
1441 }
1442
Erik Språngd7329ca2019-02-21 21:19:53 +01001443 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001444 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001445 num_expected_layers_ = num_layers;
1446 }
1447
Erik Språngb7cb7b52019-02-26 15:52:33 +01001448 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001449 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001450 return last_capture_time_ms_;
1451 }
1452
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001453 const EncodedImage& GetLastEncodedImage() {
1454 MutexLock lock(&mutex_);
1455 return last_encoded_image_;
1456 }
1457
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001458 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001459 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001460 return std::move(last_encoded_image_data_);
1461 }
1462
Per Kjellanderdcef6412020-10-07 15:09:05 +02001463 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1464 MutexLock lock(&mutex_);
1465 return last_bitrate_allocation_;
1466 }
1467
1468 int number_of_bitrate_allocations() const {
1469 MutexLock lock(&mutex_);
1470 return number_of_bitrate_allocations_;
1471 }
1472
Per Kjellandera9434842020-10-15 17:53:22 +02001473 VideoLayersAllocation GetLastVideoLayersAllocation() {
1474 MutexLock lock(&mutex_);
1475 return last_layers_allocation_;
1476 }
1477
1478 int number_of_layers_allocations() const {
1479 MutexLock lock(&mutex_);
1480 return number_of_layers_allocations_;
1481 }
1482
perkj26091b12016-09-01 01:17:40 -07001483 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001484 Result OnEncodedImage(
1485 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001486 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001487 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001488 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001489 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001490 last_encoded_image_data_ = std::vector<uint8_t>(
1491 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001492 uint32_t timestamp = encoded_image.Timestamp();
1493 if (last_timestamp_ != timestamp) {
1494 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001495 last_width_ = encoded_image._encodedWidth;
1496 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001497 } else {
1498 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001499 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1500 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001501 }
1502 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001503 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001504 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001505 if (num_received_layers_ == num_expected_layers_) {
1506 encoded_frame_event_.Set();
1507 }
sprangb1ca0732017-02-01 08:38:12 -08001508 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001509 }
1510
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001511 void OnEncoderConfigurationChanged(
1512 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001513 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001514 VideoEncoderConfig::ContentType content_type,
1515 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001516 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001517 ++number_of_reconfigurations_;
1518 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1519 }
1520
Per Kjellanderdcef6412020-10-07 15:09:05 +02001521 void OnBitrateAllocationUpdated(
1522 const VideoBitrateAllocation& allocation) override {
1523 MutexLock lock(&mutex_);
1524 ++number_of_bitrate_allocations_;
1525 last_bitrate_allocation_ = allocation;
1526 }
1527
Per Kjellandera9434842020-10-15 17:53:22 +02001528 void OnVideoLayersAllocationUpdated(
1529 VideoLayersAllocation allocation) override {
1530 MutexLock lock(&mutex_);
1531 ++number_of_layers_allocations_;
1532 last_layers_allocation_ = allocation;
1533 rtc::StringBuilder log;
1534 for (const auto& layer : allocation.active_spatial_layers) {
1535 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1536 << "[";
1537 for (const auto target_bitrate :
1538 layer.target_bitrate_per_temporal_layer) {
1539 log << target_bitrate.kbps() << ",";
1540 }
1541 log << "]";
1542 }
Harald Alvestrand97597c02021-11-04 12:01:23 +00001543 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 17:53:22 +02001544 }
1545
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001546 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001547 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001548 TestEncoder* test_encoder_;
1549 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001550 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001551 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001552 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001553 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001554 uint32_t last_height_ = 0;
1555 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001556 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001557 size_t num_expected_layers_ = 1;
1558 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001559 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001560 int number_of_reconfigurations_ = 0;
1561 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001562 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1563 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001564 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1565 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001566 };
1567
Sergey Silkin5ee69672019-07-02 14:18:34 +02001568 class VideoBitrateAllocatorProxyFactory
1569 : public VideoBitrateAllocatorFactory {
1570 public:
1571 VideoBitrateAllocatorProxyFactory()
1572 : bitrate_allocator_factory_(
1573 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1574
1575 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1576 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001577 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001578 codec_config_ = codec;
1579 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1580 }
1581
1582 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001583 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001584 return codec_config_;
1585 }
1586
1587 private:
1588 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1589
Markus Handella3765182020-07-08 13:13:32 +02001590 mutable Mutex mutex_;
1591 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001592 };
1593
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001594 Clock* clock() { return time_controller_.GetClock(); }
1595 void AdvanceTime(TimeDelta duration) {
1596 time_controller_.AdvanceTime(duration);
1597 }
1598
1599 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1600
1601 protected:
1602 virtual TaskQueueFactory* GetTaskQueueFactory() {
1603 return time_controller_.GetTaskQueueFactory();
1604 }
1605
Jonas Orelandc7f691a2022-03-09 15:12:07 +01001606 test::ScopedKeyValueConfig field_trials_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001607 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001608 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001609 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001610 int codec_width_;
1611 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001612 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001613 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001614 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001615 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001616 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001617 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001618 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001619 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001620};
1621
mflodmancc3d4422017-08-03 08:27:51 -07001622TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001623 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001624 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001625 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001626 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001627 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001628 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001629 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001630}
1631
mflodmancc3d4422017-08-03 08:27:51 -07001632TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001633 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001634 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001635 // The encoder will cache up to one frame for a short duration. Adding two
1636 // frames means that the first frame will be dropped and the second frame will
1637 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001638 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001639 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001640 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Markus Handell28c71802021-11-08 10:11:55 +01001641 AdvanceTime(TimeDelta::Zero());
perkja49cbd32016-09-16 07:53:41 -07001642 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001643
Henrik Boström381d1092020-05-12 18:49:07 +02001644 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001645 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001646
Sebastian Janssona3177052018-04-10 13:05:49 +02001647 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001648 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001649 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1650
1651 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001652 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001653}
1654
mflodmancc3d4422017-08-03 08:27:51 -07001655TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001656 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001657 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001658 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001659 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001660
Henrik Boström381d1092020-05-12 18:49:07 +02001661 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001662 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1663
Sebastian Janssona3177052018-04-10 13:05:49 +02001664 // The encoder will cache up to one frame for a short duration. Adding two
1665 // frames means that the first frame will be dropped and the second frame will
1666 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001667 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001668 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001669
Henrik Boström381d1092020-05-12 18:49:07 +02001670 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001671 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001672 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001673 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1674 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001675 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001676}
1677
mflodmancc3d4422017-08-03 08:27:51 -07001678TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001679 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001680 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001681 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001682 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001683
1684 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001685 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001686
perkja49cbd32016-09-16 07:53:41 -07001687 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001688 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001689 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001690}
1691
mflodmancc3d4422017-08-03 08:27:51 -07001692TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001693 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001694 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001695
perkja49cbd32016-09-16 07:53:41 -07001696 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001697 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001698
mflodmancc3d4422017-08-03 08:27:51 -07001699 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001700 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001701 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001702 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1703 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001704}
1705
Markus Handell9a478b52021-11-18 16:07:01 +01001706TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
1707 test::FrameForwarder source;
1708 video_stream_encoder_->SetSource(&source,
1709 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02001710 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001711 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001712
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001713 int dropped_count = 0;
1714 stats_proxy_->SetDroppedFrameCallback(
1715 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1716 ++dropped_count;
1717 });
1718
Markus Handell9a478b52021-11-18 16:07:01 +01001719 source.IncomingCapturedFrame(CreateFrame(1, nullptr));
1720 source.IncomingCapturedFrame(CreateFrame(2, nullptr));
1721 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001722 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001723 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001724}
1725
Henrik Boström56db9ff2021-03-24 09:06:45 +01001726TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001727 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001728 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001729
1730 rtc::Event frame_destroyed_event;
1731 video_source_.IncomingCapturedFrame(
1732 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001733 WaitForEncodedFrame(1);
1734 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1735 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001736 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1737 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001738 video_stream_encoder_->Stop();
1739}
1740
Henrik Boström56db9ff2021-03-24 09:06:45 +01001741TEST_F(VideoStreamEncoderTest,
1742 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001743 // Use the cropping factory.
1744 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001745 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001746 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1747 kMaxPayloadLength);
1748 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1749
1750 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001751 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001752 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001753 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1754 WaitForEncodedFrame(1);
1755 // The encoder will have been configured once.
1756 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001757 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1758 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 06:59:12 -07001759
1760 // Now send in a fake frame that needs to be cropped as the width/height
1761 // aren't divisible by 4 (see CreateEncoderStreams above).
1762 rtc::Event frame_destroyed_event;
1763 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1764 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001765 WaitForEncodedFrame(2);
1766 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1767 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001768 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1769 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001770 video_stream_encoder_->Stop();
1771}
1772
Evan Shrubsole895556e2020-10-05 09:15:13 +02001773TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1774 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001775 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001776
1777 video_source_.IncomingCapturedFrame(
1778 CreateNV12Frame(1, codec_width_, codec_height_));
1779 WaitForEncodedFrame(1);
1780 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1781 fake_encoder_.GetLastInputPixelFormat());
1782 video_stream_encoder_->Stop();
1783}
1784
Henrik Boström56db9ff2021-03-24 09:06:45 +01001785TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001786 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001787 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001788
1789 fake_encoder_.SetPreferredPixelFormats({});
1790
1791 rtc::Event frame_destroyed_event;
1792 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1793 1, &frame_destroyed_event, codec_width_, codec_height_));
1794 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001795 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001796 fake_encoder_.GetLastInputPixelFormat());
1797 video_stream_encoder_->Stop();
1798}
1799
Henrik Boström56db9ff2021-03-24 09:06:45 +01001800TEST_F(VideoStreamEncoderTest,
1801 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001802 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001803 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001804
1805 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1806
1807 rtc::Event frame_destroyed_event;
1808 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1809 1, &frame_destroyed_event, codec_width_, codec_height_));
1810 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001811 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001812 fake_encoder_.GetLastInputPixelFormat());
1813 video_stream_encoder_->Stop();
1814}
1815
Henrik Boström56db9ff2021-03-24 09:06:45 +01001816TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001817 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001818 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001819
1820 // Fake NV12 native frame does not allow mapping to I444.
1821 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1822
1823 rtc::Event frame_destroyed_event;
1824 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1825 1, &frame_destroyed_event, codec_width_, codec_height_));
1826 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001827 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001828 fake_encoder_.GetLastInputPixelFormat());
1829 video_stream_encoder_->Stop();
1830}
1831
Henrik Boström56db9ff2021-03-24 09:06:45 +01001832TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001833 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001834 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001835
1836 rtc::Event frame_destroyed_event;
1837 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1838 1, &frame_destroyed_event, codec_width_, codec_height_));
1839 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001840 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001841 fake_encoder_.GetLastInputPixelFormat());
1842 video_stream_encoder_->Stop();
1843}
1844
Ying Wang9b881ab2020-02-07 14:29:32 +01001845TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001846 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001847 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001848 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1849 WaitForEncodedFrame(1);
1850
Henrik Boström381d1092020-05-12 18:49:07 +02001851 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001852 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001853 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1854 // frames. Adding two frames means that the first frame will be dropped and
1855 // the second frame will be sent to the encoder.
1856 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1857 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1858 WaitForEncodedFrame(3);
1859 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1860 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1861 WaitForEncodedFrame(5);
1862 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1863 video_stream_encoder_->Stop();
1864}
1865
mflodmancc3d4422017-08-03 08:27:51 -07001866TEST_F(VideoStreamEncoderTest,
1867 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001868 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001869 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001870 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001871
1872 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001873 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001874 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001875 // The encoder will have been configured once when the first frame is
1876 // received.
1877 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001878
1879 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001880 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001881 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001882 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001883 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001884
1885 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001886 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001887 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001888 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001889 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001890
mflodmancc3d4422017-08-03 08:27:51 -07001891 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001892}
1893
mflodmancc3d4422017-08-03 08:27:51 -07001894TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001895 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001896 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001897
1898 // Capture a frame and wait for it to synchronize with the encoder thread.
1899 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001900 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001901 // The encoder will have been configured once.
1902 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001903 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1904 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-02 23:45:26 -07001905
1906 codec_width_ *= 2;
1907 codec_height_ *= 2;
1908 // Capture a frame with a higher resolution and wait for it to synchronize
1909 // with the encoder thread.
1910 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001911 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 10:07:11 +02001912 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1913 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 21:37:57 +01001914 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001915
mflodmancc3d4422017-08-03 08:27:51 -07001916 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001917}
1918
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001919TEST_F(VideoStreamEncoderTest,
1920 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001921 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001922 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001923
1924 // Capture a frame and wait for it to synchronize with the encoder thread.
1925 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1926 WaitForEncodedFrame(1);
1927
1928 VideoEncoderConfig video_encoder_config;
1929 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1930 // Changing the max payload data length recreates encoder.
1931 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1932 kMaxPayloadLength / 2);
1933
1934 // Capture a frame and wait for it to synchronize with the encoder thread.
1935 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1936 WaitForEncodedFrame(2);
1937 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1938
1939 video_stream_encoder_->Stop();
1940}
1941
Sergey Silkin5ee69672019-07-02 14:18:34 +02001942TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001943 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001944 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001945
1946 VideoEncoderConfig video_encoder_config;
1947 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02001948 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1949 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001950 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1951 kMaxPayloadLength);
1952
1953 // Capture a frame and wait for it to synchronize with the encoder thread.
1954 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1955 WaitForEncodedFrame(1);
1956 // The encoder will have been configured once when the first frame is
1957 // received.
1958 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001959 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001960 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001961 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001962 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1963
Sergey Silkin6456e352019-07-08 17:56:40 +02001964 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1965 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 10:07:11 +02001966 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1967 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001968 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1969 kMaxPayloadLength);
1970
1971 // Capture a frame and wait for it to synchronize with the encoder thread.
1972 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1973 WaitForEncodedFrame(2);
1974 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1975 // Bitrate limits have changed - rate allocator should be reconfigured,
1976 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 10:07:11 +02001977 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001978 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001979 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001980 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001981 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001982
1983 video_stream_encoder_->Stop();
1984}
1985
Sergey Silkin6456e352019-07-08 17:56:40 +02001986TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001987 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001988 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001989 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001990
Sergey Silkincd02eba2020-01-20 14:48:40 +01001991 const uint32_t kMinEncBitrateKbps = 100;
1992 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001993 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001994 /*frame_size_pixels=*/codec_width_ * codec_height_,
1995 /*min_start_bitrate_bps=*/0,
1996 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1997 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001998 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1999
Sergey Silkincd02eba2020-01-20 14:48:40 +01002000 VideoEncoderConfig video_encoder_config;
2001 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2002 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
2003 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2004 (kMinEncBitrateKbps + 1) * 1000;
2005 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2006 kMaxPayloadLength);
2007
2008 // When both encoder and app provide bitrate limits, the intersection of
2009 // provided sets should be used.
2010 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2011 WaitForEncodedFrame(1);
2012 EXPECT_EQ(kMaxEncBitrateKbps,
2013 bitrate_allocator_factory_.codec_config().maxBitrate);
2014 EXPECT_EQ(kMinEncBitrateKbps + 1,
2015 bitrate_allocator_factory_.codec_config().minBitrate);
2016
2017 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
2018 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2019 (kMinEncBitrateKbps - 1) * 1000;
2020 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2021 kMaxPayloadLength);
2022 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002023 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002024 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002025 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002026 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002027 bitrate_allocator_factory_.codec_config().minBitrate);
2028
Sergey Silkincd02eba2020-01-20 14:48:40 +01002029 video_stream_encoder_->Stop();
2030}
2031
2032TEST_F(VideoStreamEncoderTest,
2033 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02002034 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002035 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002036
2037 const uint32_t kMinAppBitrateKbps = 100;
2038 const uint32_t kMaxAppBitrateKbps = 200;
2039 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
2040 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
2041 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2042 /*frame_size_pixels=*/codec_width_ * codec_height_,
2043 /*min_start_bitrate_bps=*/0,
2044 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
2045 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
2046 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2047
2048 VideoEncoderConfig video_encoder_config;
2049 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2050 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
2051 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2052 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002053 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2054 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002055
Sergey Silkincd02eba2020-01-20 14:48:40 +01002056 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2057 WaitForEncodedFrame(1);
2058 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002059 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002060 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002061 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02002062
2063 video_stream_encoder_->Stop();
2064}
2065
2066TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002067 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02002068 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002069 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02002070
2071 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002072 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002073 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002074 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002075 fake_encoder_.SetResolutionBitrateLimits(
2076 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
2077
2078 VideoEncoderConfig video_encoder_config;
2079 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2080 video_encoder_config.max_bitrate_bps = 0;
2081 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2082 kMaxPayloadLength);
2083
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002084 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002085 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2086 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002087 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2088 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002089 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2090 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2091
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002092 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002093 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2094 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002095 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2096 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002097 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2098 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2099
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002100 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02002101 // encoder for 360p should be used.
2102 video_source_.IncomingCapturedFrame(
2103 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2104 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002105 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2106 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002107 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2108 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2109
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002110 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02002111 // ignored.
2112 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2113 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002114 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2115 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002116 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2117 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002118 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2119 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002120 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2121 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2122
2123 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2124 // for 270p should be used.
2125 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2126 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002127 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2128 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002129 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2130 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2131
2132 video_stream_encoder_->Stop();
2133}
2134
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002135TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002136 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002137 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002138
2139 VideoEncoderConfig video_encoder_config;
2140 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2141 video_encoder_config.max_bitrate_bps = 0;
2142 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2143 kMaxPayloadLength);
2144
2145 // Encode 720p frame to get the default encoder target bitrate.
2146 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2147 WaitForEncodedFrame(1);
2148 const uint32_t kDefaultTargetBitrateFor720pKbps =
2149 bitrate_allocator_factory_.codec_config()
2150 .simulcastStream[0]
2151 .targetBitrate;
2152
2153 // Set the max recommended encoder bitrate to something lower than the default
2154 // target bitrate.
2155 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2156 1280 * 720, 10 * 1000, 10 * 1000,
2157 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2158 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2159
2160 // Change resolution to trigger encoder reinitialization.
2161 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2162 WaitForEncodedFrame(2);
2163 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2164 WaitForEncodedFrame(3);
2165
2166 // Ensure the target bitrate is capped by the max bitrate.
2167 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2168 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2169 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2170 .simulcastStream[0]
2171 .targetBitrate *
2172 1000,
2173 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2174
2175 video_stream_encoder_->Stop();
2176}
2177
Åsa Perssona7e34d32021-01-20 15:36:13 +01002178TEST_F(VideoStreamEncoderTest,
2179 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2180 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2181 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2182 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2183 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2184 fake_encoder_.SetResolutionBitrateLimits(
2185 {kEncoderLimits270p, kEncoderLimits360p});
2186
2187 // Two streams, highest stream active.
2188 VideoEncoderConfig config;
2189 const int kNumStreams = 2;
2190 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2191 config.max_bitrate_bps = 0;
2192 config.simulcast_layers[0].active = false;
2193 config.simulcast_layers[1].active = true;
2194 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002195 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002196 "VP8", /*max qp*/ 56, /*screencast*/ false,
2197 /*screenshare enabled*/ false);
2198 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2199
2200 // The encoder bitrate limits for 270p should be used.
2201 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2202 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002203 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002204 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002205 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002206 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002207 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002208
2209 // The encoder bitrate limits for 360p should be used.
2210 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2211 EXPECT_FALSE(WaitForFrame(1000));
2212 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002213 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002214 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002215 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002216
2217 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2218 video_source_.IncomingCapturedFrame(
2219 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2220 EXPECT_FALSE(WaitForFrame(1000));
2221 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002222 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002223 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002224 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002225
2226 // Resolution higher than 360p. Encoder limits should be ignored.
2227 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2228 EXPECT_FALSE(WaitForFrame(1000));
2229 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002230 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002231 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002232 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002233 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002234 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002235 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002236 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002237
2238 // Resolution lower than 270p. The encoder limits for 270p should be used.
2239 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2240 EXPECT_FALSE(WaitForFrame(1000));
2241 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002242 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002243 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002244 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002245
2246 video_stream_encoder_->Stop();
2247}
2248
2249TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002250 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2251 // Two streams, highest stream active.
2252 VideoEncoderConfig config;
2253 const int kNumStreams = 2;
2254 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2255 config.max_bitrate_bps = 0;
2256 config.simulcast_layers[0].active = false;
2257 config.simulcast_layers[1].active = true;
2258 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002259 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002260 "VP8", /*max qp*/ 56, /*screencast*/ false,
2261 /*screenshare enabled*/ false);
2262 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2263
2264 // Default bitrate limits for 270p should be used.
2265 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2266 kDefaultLimits270p =
2267 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002268 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002269 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2270 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002271 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 10:39:51 +01002272 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002273 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002274 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002275 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002276
2277 // Default bitrate limits for 360p should be used.
2278 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2279 kDefaultLimits360p =
2280 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002281 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002282 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2283 EXPECT_FALSE(WaitForFrame(1000));
2284 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002285 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002286 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002287 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002288
2289 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2290 video_source_.IncomingCapturedFrame(
2291 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2292 EXPECT_FALSE(WaitForFrame(1000));
2293 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002294 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002295 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002296 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002297
2298 // Default bitrate limits for 540p should be used.
2299 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2300 kDefaultLimits540p =
2301 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002302 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002303 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2304 EXPECT_FALSE(WaitForFrame(1000));
2305 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002306 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002307 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002308 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002309
2310 video_stream_encoder_->Stop();
2311}
2312
2313TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002314 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2315 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2316 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2317 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2318 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2319 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2320 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2321 fake_encoder_.SetResolutionBitrateLimits(
2322 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2323
2324 // Three streams, middle stream active.
2325 VideoEncoderConfig config;
2326 const int kNumStreams = 3;
2327 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2328 config.simulcast_layers[0].active = false;
2329 config.simulcast_layers[1].active = true;
2330 config.simulcast_layers[2].active = false;
2331 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002332 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002333 "VP8", /*max qp*/ 56, /*screencast*/ false,
2334 /*screenshare enabled*/ false);
2335 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2336
2337 // The encoder bitrate limits for 360p should be used.
2338 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2339 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002340 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002341 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002342 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002343 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002344 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002345
2346 // The encoder bitrate limits for 270p should be used.
2347 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2348 EXPECT_FALSE(WaitForFrame(1000));
2349 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002350 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002351 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002352 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002353
2354 video_stream_encoder_->Stop();
2355}
2356
2357TEST_F(VideoStreamEncoderTest,
2358 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2359 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2360 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2361 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2362 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2363 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2364 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2365 fake_encoder_.SetResolutionBitrateLimits(
2366 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2367
2368 // Three streams, lowest stream active.
2369 VideoEncoderConfig config;
2370 const int kNumStreams = 3;
2371 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2372 config.simulcast_layers[0].active = true;
2373 config.simulcast_layers[1].active = false;
2374 config.simulcast_layers[2].active = false;
2375 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002376 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002377 "VP8", /*max qp*/ 56, /*screencast*/ false,
2378 /*screenshare enabled*/ false);
2379 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2380
2381 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2382 // on lowest stream, limits for 270p should not be used
2383 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2384 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002385 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002386 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002387 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002388 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002389 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002390
2391 video_stream_encoder_->Stop();
2392}
2393
2394TEST_F(VideoStreamEncoderTest,
2395 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2396 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2397 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2398 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2399 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2400 fake_encoder_.SetResolutionBitrateLimits(
2401 {kEncoderLimits270p, kEncoderLimits360p});
2402 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2403
2404 // Two streams, highest stream active.
2405 VideoEncoderConfig config;
2406 const int kNumStreams = 2;
2407 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2408 config.simulcast_layers[0].active = false;
2409 config.simulcast_layers[1].active = true;
2410 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2411 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002412 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002413 "VP8", /*max qp*/ 56, /*screencast*/ false,
2414 /*screenshare enabled*/ false);
2415 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2416
2417 // The encoder bitrate limits for 270p should be used.
2418 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2419 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002420 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002421 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002422 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002423 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002424 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002425
2426 // The max configured bitrate is less than the encoder limit for 360p.
2427 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2428 EXPECT_FALSE(WaitForFrame(1000));
2429 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002430 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002431 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002432 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002433
2434 video_stream_encoder_->Stop();
2435}
2436
mflodmancc3d4422017-08-03 08:27:51 -07002437TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002438 EXPECT_TRUE(video_source_.has_sinks());
2439 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002440 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002441 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002442 EXPECT_FALSE(video_source_.has_sinks());
2443 EXPECT_TRUE(new_video_source.has_sinks());
2444
mflodmancc3d4422017-08-03 08:27:51 -07002445 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002446}
2447
mflodmancc3d4422017-08-03 08:27:51 -07002448TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002449 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002450 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002451 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002452 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002453}
2454
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002455class ResolutionAlignmentTest
2456 : public VideoStreamEncoderTest,
2457 public ::testing::WithParamInterface<
2458 ::testing::tuple<int, std::vector<double>>> {
2459 public:
2460 ResolutionAlignmentTest()
2461 : requested_alignment_(::testing::get<0>(GetParam())),
2462 scale_factors_(::testing::get<1>(GetParam())) {}
2463
2464 protected:
2465 const int requested_alignment_;
2466 const std::vector<double> scale_factors_;
2467};
2468
2469INSTANTIATE_TEST_SUITE_P(
2470 AlignmentAndScaleFactors,
2471 ResolutionAlignmentTest,
2472 ::testing::Combine(
2473 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2474 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2475 std::vector<double>{-1.0, -1.0},
2476 std::vector<double>{-1.0, -1.0, -1.0},
2477 std::vector<double>{4.0, 2.0, 1.0},
2478 std::vector<double>{9999.0, -1.0, 1.0},
2479 std::vector<double>{3.99, 2.01, 1.0},
2480 std::vector<double>{4.9, 1.7, 1.25},
2481 std::vector<double>{10.0, 4.0, 3.0},
2482 std::vector<double>{1.75, 3.5},
2483 std::vector<double>{1.5, 2.5},
2484 std::vector<double>{1.3, 1.0})));
2485
2486TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2487 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002488 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002489 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2490 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2491
2492 // Fill config with the scaling factor by which to reduce encoding size.
2493 const int num_streams = scale_factors_.size();
2494 VideoEncoderConfig config;
2495 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2496 for (int i = 0; i < num_streams; ++i) {
2497 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2498 }
2499 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002500 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002501 "VP8", /*max qp*/ 56, /*screencast*/ false,
2502 /*screenshare enabled*/ false);
2503 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2504
Henrik Boström381d1092020-05-12 18:49:07 +02002505 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002506 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2507 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002508 // Wait for all layers before triggering event.
2509 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002510
2511 // On the 1st frame, we should have initialized the encoder and
2512 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002513 int64_t timestamp_ms = kFrameIntervalMs;
2514 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2515 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002516 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002517
2518 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2519 // (It's up the to the encoder to potentially drop the previous frame,
2520 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002521 timestamp_ms += kFrameIntervalMs;
2522 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2523 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002524 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002525
Asa Persson606d3cb2021-10-04 10:07:11 +02002526 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002527 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2528 // Frame size should be a multiple of the requested alignment.
2529 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2530 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2531 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2532 // Aspect ratio should match.
2533 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2534 codec.height * codec.simulcastStream[i].width);
2535 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002536
2537 video_stream_encoder_->Stop();
2538}
2539
Jonathan Yubc771b72017-12-08 17:04:29 -08002540TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2541 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002542 const int kWidth = 1280;
2543 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002544
2545 // We rely on the automatic resolution adaptation, but we handle framerate
2546 // adaptation manually by mocking the stats proxy.
2547 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002548
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002549 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002550 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002551 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002552 video_stream_encoder_->SetSource(&video_source_,
2553 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002554 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002555 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002556 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002557 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2558
Jonathan Yubc771b72017-12-08 17:04:29 -08002559 // Adapt down as far as possible.
2560 rtc::VideoSinkWants last_wants;
2561 int64_t t = 1;
2562 int loop_count = 0;
2563 do {
2564 ++loop_count;
2565 last_wants = video_source_.sink_wants();
2566
2567 // Simulate the framerate we've been asked to adapt to.
2568 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2569 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2570 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2571 mock_stats.input_frame_rate = fps;
2572 stats_proxy_->SetMockStats(mock_stats);
2573
2574 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2575 sink_.WaitForEncodedFrame(t);
2576 t += frame_interval_ms;
2577
mflodmancc3d4422017-08-03 08:27:51 -07002578 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002579 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002580 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002581 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2582 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002583 } while (video_source_.sink_wants().max_pixel_count <
2584 last_wants.max_pixel_count ||
2585 video_source_.sink_wants().max_framerate_fps <
2586 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002587
Jonathan Yubc771b72017-12-08 17:04:29 -08002588 // Verify that we've adapted all the way down.
2589 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002590 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002591 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2592 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002593 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002594 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2595 *video_source_.last_sent_height());
2596 EXPECT_EQ(kMinBalancedFramerateFps,
2597 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002598
Jonathan Yubc771b72017-12-08 17:04:29 -08002599 // Adapt back up the same number of times we adapted down.
2600 for (int i = 0; i < loop_count - 1; ++i) {
2601 last_wants = video_source_.sink_wants();
2602
2603 // Simulate the framerate we've been asked to adapt to.
2604 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2605 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2606 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2607 mock_stats.input_frame_rate = fps;
2608 stats_proxy_->SetMockStats(mock_stats);
2609
2610 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2611 sink_.WaitForEncodedFrame(t);
2612 t += frame_interval_ms;
2613
Henrik Boström91aa7322020-04-28 12:24:33 +02002614 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002615 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002616 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002617 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2618 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002619 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2620 last_wants.max_pixel_count ||
2621 video_source_.sink_wants().max_framerate_fps >
2622 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002623 }
2624
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002625 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002626 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002627 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002628 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2629 EXPECT_EQ((loop_count - 1) * 2,
2630 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002631
mflodmancc3d4422017-08-03 08:27:51 -07002632 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002633}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002634
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002635TEST_F(VideoStreamEncoderTest,
2636 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02002637 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2638 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002639 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002640
2641 const int kFrameWidth = 1280;
2642 const int kFrameHeight = 720;
2643
2644 int64_t ntp_time = kFrameIntervalMs;
2645
2646 // Force an input frame rate to be available, or the adaptation call won't
2647 // know what framerate to adapt form.
2648 const int kInputFps = 30;
2649 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2650 stats.input_frame_rate = kInputFps;
2651 stats_proxy_->SetMockStats(stats);
2652
2653 video_source_.set_adaptation_enabled(true);
2654 video_stream_encoder_->SetSource(
2655 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002656 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002657 video_source_.IncomingCapturedFrame(
2658 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2659 sink_.WaitForEncodedFrame(ntp_time);
2660 ntp_time += kFrameIntervalMs;
2661
2662 // Trigger CPU overuse.
2663 video_stream_encoder_->TriggerCpuOveruse();
2664 video_source_.IncomingCapturedFrame(
2665 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2666 sink_.WaitForEncodedFrame(ntp_time);
2667 ntp_time += kFrameIntervalMs;
2668
2669 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2670 EXPECT_EQ(std::numeric_limits<int>::max(),
2671 video_source_.sink_wants().max_pixel_count);
2672 // Some framerate constraint should be set.
2673 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2674 EXPECT_LT(restricted_fps, kInputFps);
2675 video_source_.IncomingCapturedFrame(
2676 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2677 sink_.WaitForEncodedFrame(ntp_time);
2678 ntp_time += 100;
2679
Henrik Boström2671dac2020-05-19 16:29:09 +02002680 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002681 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2682 // Give the encoder queue time to process the change in degradation preference
2683 // by waiting for an encoded frame.
2684 video_source_.IncomingCapturedFrame(
2685 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2686 sink_.WaitForEncodedFrame(ntp_time);
2687 ntp_time += kFrameIntervalMs;
2688
2689 video_stream_encoder_->TriggerQualityLow();
2690 video_source_.IncomingCapturedFrame(
2691 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2692 sink_.WaitForEncodedFrame(ntp_time);
2693 ntp_time += kFrameIntervalMs;
2694
2695 // Some resolution constraint should be set.
2696 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2697 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2698 kFrameWidth * kFrameHeight);
2699 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2700
2701 int pixel_count = video_source_.sink_wants().max_pixel_count;
2702 // Triggering a CPU underuse should not change the sink wants since it has
2703 // not been overused for resolution since we changed degradation preference.
2704 video_stream_encoder_->TriggerCpuUnderuse();
2705 video_source_.IncomingCapturedFrame(
2706 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2707 sink_.WaitForEncodedFrame(ntp_time);
2708 ntp_time += kFrameIntervalMs;
2709 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2710 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2711
Evan Shrubsole64469032020-06-11 10:45:29 +02002712 // Change the degradation preference back. CPU underuse should not adapt since
2713 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002714 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002715 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2716 video_source_.IncomingCapturedFrame(
2717 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2718 sink_.WaitForEncodedFrame(ntp_time);
2719 ntp_time += 100;
2720 // Resolution adaptations is gone after changing degradation preference.
2721 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2722 EXPECT_EQ(std::numeric_limits<int>::max(),
2723 video_source_.sink_wants().max_pixel_count);
2724 // The fps adaptation from above is now back.
2725 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2726
2727 // Trigger CPU underuse.
2728 video_stream_encoder_->TriggerCpuUnderuse();
2729 video_source_.IncomingCapturedFrame(
2730 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2731 sink_.WaitForEncodedFrame(ntp_time);
2732 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002733 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2734
2735 // Trigger QP underuse, fps should return to normal.
2736 video_stream_encoder_->TriggerQualityHigh();
2737 video_source_.IncomingCapturedFrame(
2738 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2739 sink_.WaitForEncodedFrame(ntp_time);
2740 ntp_time += kFrameIntervalMs;
2741 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002742
2743 video_stream_encoder_->Stop();
2744}
2745
mflodmancc3d4422017-08-03 08:27:51 -07002746TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002747 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002748 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002749 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002750
sprangc5d62e22017-04-02 23:53:04 -07002751 const int kFrameWidth = 1280;
2752 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002753
Åsa Persson8c1bf952018-09-13 10:42:19 +02002754 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002755
kthelgason5e13d412016-12-01 03:59:51 -08002756 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002757 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002758 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002759 frame_timestamp += kFrameIntervalMs;
2760
perkj803d97f2016-11-01 11:45:46 -07002761 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002762 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002763 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002764 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002765 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002766 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002767
asapersson0944a802017-04-07 00:57:58 -07002768 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002769 // wanted resolution.
2770 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2771 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2772 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002773 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002774
2775 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002776 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002777 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002778 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002779 // Give the encoder queue time to process the change in degradation preference
2780 // by waiting for an encoded frame.
2781 new_video_source.IncomingCapturedFrame(
2782 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2783 sink_.WaitForEncodedFrame(frame_timestamp);
2784 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002785 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002786 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002787
sprangc5d62e22017-04-02 23:53:04 -07002788 // Force an input frame rate to be available, or the adaptation call won't
2789 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002790 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002791 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002792 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002793 stats_proxy_->SetMockStats(stats);
2794
mflodmancc3d4422017-08-03 08:27:51 -07002795 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002796 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002797 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002798 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002799 frame_timestamp += kFrameIntervalMs;
2800
2801 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002802 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002803 EXPECT_EQ(std::numeric_limits<int>::max(),
2804 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002805 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002806
asapersson02465b82017-04-10 01:12:52 -07002807 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002808 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2809 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002810 // Give the encoder queue time to process the change in degradation preference
2811 // by waiting for an encoded frame.
2812 new_video_source.IncomingCapturedFrame(
2813 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2814 sink_.WaitForEncodedFrame(frame_timestamp);
2815 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002816 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002817
mflodmancc3d4422017-08-03 08:27:51 -07002818 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002819 new_video_source.IncomingCapturedFrame(
2820 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002821 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002822 frame_timestamp += kFrameIntervalMs;
2823
2824 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002825 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002826
2827 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002828 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002829 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002830 // Give the encoder queue time to process the change in degradation preference
2831 // by waiting for an encoded frame.
2832 new_video_source.IncomingCapturedFrame(
2833 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2834 sink_.WaitForEncodedFrame(frame_timestamp);
2835 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002836 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2837 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002838 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002839 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002840
2841 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002842 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002843 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002844 // Give the encoder queue time to process the change in degradation preference
2845 // by waiting for an encoded frame.
2846 new_video_source.IncomingCapturedFrame(
2847 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2848 sink_.WaitForEncodedFrame(frame_timestamp);
2849 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002850 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2851 EXPECT_EQ(std::numeric_limits<int>::max(),
2852 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002853 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002854
mflodmancc3d4422017-08-03 08:27:51 -07002855 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002856}
2857
mflodmancc3d4422017-08-03 08:27:51 -07002858TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002859 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002860 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002861
asaperssonfab67072017-04-04 05:51:49 -07002862 const int kWidth = 1280;
2863 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002864 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002865 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002866 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2867 EXPECT_FALSE(stats.bw_limited_resolution);
2868 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2869
2870 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002871 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002872 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002873 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002874
2875 stats = stats_proxy_->GetStats();
2876 EXPECT_TRUE(stats.bw_limited_resolution);
2877 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2878
2879 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002880 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002881 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002882 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002883
2884 stats = stats_proxy_->GetStats();
2885 EXPECT_FALSE(stats.bw_limited_resolution);
2886 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2887 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2888
mflodmancc3d4422017-08-03 08:27:51 -07002889 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002890}
2891
mflodmancc3d4422017-08-03 08:27:51 -07002892TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002893 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002894 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002895
2896 const int kWidth = 1280;
2897 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002898 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002899 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002900 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2901 EXPECT_FALSE(stats.cpu_limited_resolution);
2902 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2903
2904 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002905 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002906 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002907 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002908
2909 stats = stats_proxy_->GetStats();
2910 EXPECT_TRUE(stats.cpu_limited_resolution);
2911 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2912
2913 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002914 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002915 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002916 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002917
2918 stats = stats_proxy_->GetStats();
2919 EXPECT_FALSE(stats.cpu_limited_resolution);
2920 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002921 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002922
mflodmancc3d4422017-08-03 08:27:51 -07002923 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002924}
2925
mflodmancc3d4422017-08-03 08:27:51 -07002926TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002927 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002928 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002929
asaperssonfab67072017-04-04 05:51:49 -07002930 const int kWidth = 1280;
2931 const int kHeight = 720;
2932 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002933 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002934 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002935 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002936 EXPECT_FALSE(stats.cpu_limited_resolution);
2937 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2938
asaperssonfab67072017-04-04 05:51:49 -07002939 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002940 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002941 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002942 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002943 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002944 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002945 EXPECT_TRUE(stats.cpu_limited_resolution);
2946 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2947
2948 // Set new source with adaptation still enabled.
2949 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002950 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002951 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002952
asaperssonfab67072017-04-04 05:51:49 -07002953 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002954 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002955 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002956 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002957 EXPECT_TRUE(stats.cpu_limited_resolution);
2958 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2959
2960 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002961 video_stream_encoder_->SetSource(&new_video_source,
2962 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002963
asaperssonfab67072017-04-04 05:51:49 -07002964 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002965 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002966 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002967 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002968 EXPECT_FALSE(stats.cpu_limited_resolution);
2969 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2970
2971 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002972 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002973 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002974
asaperssonfab67072017-04-04 05:51:49 -07002975 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002976 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002977 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002978 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002979 EXPECT_TRUE(stats.cpu_limited_resolution);
2980 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2981
asaperssonfab67072017-04-04 05:51:49 -07002982 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002983 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002984 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002985 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002986 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002987 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002988 EXPECT_FALSE(stats.cpu_limited_resolution);
2989 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002990 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002991
mflodmancc3d4422017-08-03 08:27:51 -07002992 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002993}
2994
mflodmancc3d4422017-08-03 08:27:51 -07002995TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002996 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002997 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002998
asaperssonfab67072017-04-04 05:51:49 -07002999 const int kWidth = 1280;
3000 const int kHeight = 720;
3001 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003002 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003003 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003004 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003005 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003006 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003007
3008 // Set new source with adaptation still enabled.
3009 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003010 video_stream_encoder_->SetSource(&new_video_source,
3011 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08003012
asaperssonfab67072017-04-04 05:51:49 -07003013 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003014 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08003015 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003016 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003017 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003018 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003019
asaperssonfab67072017-04-04 05:51:49 -07003020 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003021 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003022 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003023 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003024 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003025 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003026 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003027 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003028
asaperssonfab67072017-04-04 05:51:49 -07003029 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003030 video_stream_encoder_->SetSource(&new_video_source,
3031 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08003032
asaperssonfab67072017-04-04 05:51:49 -07003033 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003034 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003035 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003036 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003037 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003038 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003039
asapersson02465b82017-04-10 01:12:52 -07003040 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07003041 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003042 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003043
asaperssonfab67072017-04-04 05:51:49 -07003044 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003045 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08003046 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003047 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003048 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003049 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
3050 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003051
mflodmancc3d4422017-08-03 08:27:51 -07003052 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003053}
3054
mflodmancc3d4422017-08-03 08:27:51 -07003055TEST_F(VideoStreamEncoderTest,
3056 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02003057 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003058 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07003059
3060 const int kWidth = 1280;
3061 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02003062 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07003063 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003064 video_source_.IncomingCapturedFrame(
3065 CreateFrame(timestamp_ms, kWidth, kHeight));
3066 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003067 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3068 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3069 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3070
3071 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003072 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003073 timestamp_ms += kFrameIntervalMs;
3074 video_source_.IncomingCapturedFrame(
3075 CreateFrame(timestamp_ms, kWidth, kHeight));
3076 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003077 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3078 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3079 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3080
3081 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07003082 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003083 timestamp_ms += kFrameIntervalMs;
3084 video_source_.IncomingCapturedFrame(
3085 CreateFrame(timestamp_ms, kWidth, kHeight));
3086 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003087 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3088 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3089 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3090
Niels Möller4db138e2018-04-19 09:04:13 +02003091 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07003092 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003093
3094 VideoEncoderConfig video_encoder_config;
3095 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3096 // Make format different, to force recreation of encoder.
3097 video_encoder_config.video_format.parameters["foo"] = "foo";
3098 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003099 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003100 timestamp_ms += kFrameIntervalMs;
3101 video_source_.IncomingCapturedFrame(
3102 CreateFrame(timestamp_ms, kWidth, kHeight));
3103 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003104 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3105 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3106 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3107
mflodmancc3d4422017-08-03 08:27:51 -07003108 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003109}
3110
mflodmancc3d4422017-08-03 08:27:51 -07003111TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003112 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003113 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003114 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003115
3116 const int kWidth = 1280;
3117 const int kHeight = 720;
3118 int sequence = 1;
3119
3120 // Enable BALANCED preference, no initial limitation.
3121 test::FrameForwarder source;
3122 video_stream_encoder_->SetSource(&source,
3123 webrtc::DegradationPreference::BALANCED);
3124 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3125 WaitForEncodedFrame(sequence++);
3126 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3127 EXPECT_FALSE(stats.cpu_limited_resolution);
3128 EXPECT_FALSE(stats.cpu_limited_framerate);
3129 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3130
3131 // Trigger CPU overuse, should now adapt down.
3132 video_stream_encoder_->TriggerCpuOveruse();
3133 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3134 WaitForEncodedFrame(sequence++);
3135 stats = stats_proxy_->GetStats();
3136 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3137
3138 // Set new degradation preference should clear restrictions since we changed
3139 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003140 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003141 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3142 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3143 WaitForEncodedFrame(sequence++);
3144 stats = stats_proxy_->GetStats();
3145 EXPECT_FALSE(stats.cpu_limited_resolution);
3146 EXPECT_FALSE(stats.cpu_limited_framerate);
3147 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3148
3149 // Force an input frame rate to be available, or the adaptation call won't
3150 // know what framerate to adapt from.
3151 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3152 mock_stats.input_frame_rate = 30;
3153 stats_proxy_->SetMockStats(mock_stats);
3154 video_stream_encoder_->TriggerCpuOveruse();
3155 stats_proxy_->ResetMockStats();
3156 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3157 WaitForEncodedFrame(sequence++);
3158
3159 // We have now adapted once.
3160 stats = stats_proxy_->GetStats();
3161 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3162
3163 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003164 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3165 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003166 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3167 WaitForEncodedFrame(sequence++);
3168 stats = stats_proxy_->GetStats();
3169 EXPECT_FALSE(stats.cpu_limited_resolution);
3170 EXPECT_FALSE(stats.cpu_limited_framerate);
3171 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3172
3173 video_stream_encoder_->Stop();
3174}
3175
3176TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003177 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003178 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003179 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003180
asapersson0944a802017-04-07 00:57:58 -07003181 const int kWidth = 1280;
3182 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003183 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003184
asaperssonfab67072017-04-04 05:51:49 -07003185 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003186 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003187 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003188 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003189 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003190 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3191
asapersson02465b82017-04-10 01:12:52 -07003192 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003193 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003194 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003195 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003196 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003197 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003198 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003199 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3200
3201 // Set new source with adaptation still enabled.
3202 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003203 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003204 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003205
3206 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003207 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003208 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003209 stats = stats_proxy_->GetStats();
3210 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003211 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003212 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3213
sprangc5d62e22017-04-02 23:53:04 -07003214 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003215 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003216 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003217 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003218 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003219 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003220 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003221 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003222 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003223 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003224 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3225
sprangc5d62e22017-04-02 23:53:04 -07003226 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003227 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003228 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3229 mock_stats.input_frame_rate = 30;
3230 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003231 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003232 stats_proxy_->ResetMockStats();
3233
3234 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003235 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003236 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003237
3238 // Framerate now adapted.
3239 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003240 EXPECT_FALSE(stats.cpu_limited_resolution);
3241 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003242 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3243
3244 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003245 video_stream_encoder_->SetSource(&new_video_source,
3246 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003247 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003248 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003249 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003250
3251 stats = stats_proxy_->GetStats();
3252 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003253 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003254 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3255
3256 // Try to trigger overuse. Should not succeed.
3257 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003258 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003259 stats_proxy_->ResetMockStats();
3260
3261 stats = stats_proxy_->GetStats();
3262 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003263 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003264 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3265
3266 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003267 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003268 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003269 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003270 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003271 stats = stats_proxy_->GetStats();
3272 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003273 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003274 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003275
3276 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003277 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003278 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003279 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003280 stats = stats_proxy_->GetStats();
3281 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003282 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003283 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3284
3285 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003286 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003287 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003288 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003289 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003290 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003291 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003292 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003293 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003294 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003295 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3296
3297 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003298 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003299 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003300 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003301 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003302 stats = stats_proxy_->GetStats();
3303 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003304 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003305 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003306 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003307
mflodmancc3d4422017-08-03 08:27:51 -07003308 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003309}
3310
mflodmancc3d4422017-08-03 08:27:51 -07003311TEST_F(VideoStreamEncoderTest,
3312 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003313 const int kWidth = 1280;
3314 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003315 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003316 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003317
asaperssonfab67072017-04-04 05:51:49 -07003318 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003319 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003320
asaperssonfab67072017-04-04 05:51:49 -07003321 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003322 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003323
asaperssonfab67072017-04-04 05:51:49 -07003324 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003325 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003326
asaperssonfab67072017-04-04 05:51:49 -07003327 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003328 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003329
kthelgason876222f2016-11-29 01:44:11 -08003330 // Expect a scale down.
3331 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003332 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003333
asapersson02465b82017-04-10 01:12:52 -07003334 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003335 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003336 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003337 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003338
asaperssonfab67072017-04-04 05:51:49 -07003339 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003340 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003341 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003342 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003343
asaperssonfab67072017-04-04 05:51:49 -07003344 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003345 EXPECT_EQ(std::numeric_limits<int>::max(),
3346 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003347
asaperssonfab67072017-04-04 05:51:49 -07003348 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003349 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003350 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003351 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003352
asapersson02465b82017-04-10 01:12:52 -07003353 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003354 EXPECT_EQ(std::numeric_limits<int>::max(),
3355 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003356
mflodmancc3d4422017-08-03 08:27:51 -07003357 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003358}
3359
mflodmancc3d4422017-08-03 08:27:51 -07003360TEST_F(VideoStreamEncoderTest,
3361 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003362 const int kWidth = 1280;
3363 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003364 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003365 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003366
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003367 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003368 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003369 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003370 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003371
3372 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003373 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003374 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003375 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3376 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3377
3378 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003379 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003380 EXPECT_THAT(source.sink_wants(),
3381 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003382 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3383 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3384 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3385
3386 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003387 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003388 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3389 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3390 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3391
mflodmancc3d4422017-08-03 08:27:51 -07003392 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003393}
3394
mflodmancc3d4422017-08-03 08:27:51 -07003395TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003396 const int kWidth = 1280;
3397 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003398 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003399 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003400
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003401 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003402 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003403 video_stream_encoder_->SetSource(&source,
3404 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003405 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3406 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003407 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003408
3409 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003410 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003411 EXPECT_THAT(source.sink_wants(),
3412 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003413 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3414 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3415 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3416
3417 // Trigger adapt down for same input resolution, expect no change.
3418 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3419 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003420 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003421 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3422 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3423 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3424
3425 // Trigger adapt down for larger input resolution, expect no change.
3426 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3427 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003428 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003429 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3430 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3431 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3432
mflodmancc3d4422017-08-03 08:27:51 -07003433 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003434}
3435
mflodmancc3d4422017-08-03 08:27:51 -07003436TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003437 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3438 const int kWidth = 640;
3439 const int kHeight = 360;
3440 const int64_t kFrameIntervalMs = 150;
3441 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003442 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003443
3444 // Enable BALANCED preference, no initial limitation.
3445 AdaptingFrameForwarder source(&time_controller_);
3446 source.set_adaptation_enabled(true);
3447 video_stream_encoder_->SetSource(&source,
3448 webrtc::DegradationPreference::BALANCED);
3449
3450 int64_t timestamp_ms = kFrameIntervalMs;
3451 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3452 sink_.WaitForEncodedFrame(kWidth, kHeight);
3453 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3454 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3455 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3456 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3457
3458 // Trigger adapt down, expect reduced fps (640x360@15fps).
3459 video_stream_encoder_->TriggerQualityLow();
3460 timestamp_ms += kFrameIntervalMs;
3461 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3462 sink_.WaitForEncodedFrame(timestamp_ms);
3463 EXPECT_THAT(source.sink_wants(),
3464 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3465 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3466 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3467 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3468
3469 // Source requests 270p, expect reduced resolution (480x270@15fps).
3470 source.OnOutputFormatRequest(480, 270);
3471 timestamp_ms += kFrameIntervalMs;
3472 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3473 WaitForEncodedFrame(480, 270);
3474 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3475
3476 // Trigger adapt down, expect reduced fps (480x270@10fps).
3477 video_stream_encoder_->TriggerQualityLow();
3478 timestamp_ms += kFrameIntervalMs;
3479 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3480 sink_.WaitForEncodedFrame(timestamp_ms);
3481 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3482 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3483 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3484 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3485
3486 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3487 source.OnOutputFormatRequest(320, 180);
3488 timestamp_ms += kFrameIntervalMs;
3489 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3490 WaitForEncodedFrame(320, 180);
3491 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3492
3493 // Trigger adapt down, expect reduced fps (320x180@7fps).
3494 video_stream_encoder_->TriggerQualityLow();
3495 timestamp_ms += kFrameIntervalMs;
3496 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3497 sink_.WaitForEncodedFrame(timestamp_ms);
3498 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3499 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3500 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3501 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3502
3503 // Source requests VGA, expect increased resolution (640x360@7fps).
3504 source.OnOutputFormatRequest(640, 360);
3505 timestamp_ms += kFrameIntervalMs;
3506 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3507 WaitForEncodedFrame(timestamp_ms);
3508 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3509
3510 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3511 video_stream_encoder_->TriggerQualityHigh();
3512 timestamp_ms += kFrameIntervalMs;
3513 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3514 WaitForEncodedFrame(timestamp_ms);
3515 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3516 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3517 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3518 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3519
3520 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3521 video_stream_encoder_->TriggerQualityHigh();
3522 timestamp_ms += kFrameIntervalMs;
3523 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3524 WaitForEncodedFrame(timestamp_ms);
3525 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3526 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3527 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3528 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3529
3530 // Trigger adapt up, expect increased fps (640x360@maxfps).
3531 video_stream_encoder_->TriggerQualityHigh();
3532 timestamp_ms += kFrameIntervalMs;
3533 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3534 WaitForEncodedFrame(timestamp_ms);
3535 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3536 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3537 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3538 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3539
3540 video_stream_encoder_->Stop();
3541}
3542
3543TEST_F(VideoStreamEncoderTest,
3544 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3545 const int kWidth = 1280;
3546 const int kHeight = 720;
3547 const int64_t kFrameIntervalMs = 150;
3548 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003549 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003550
3551 // Enable BALANCED preference, no initial limitation.
3552 AdaptingFrameForwarder source(&time_controller_);
3553 source.set_adaptation_enabled(true);
3554 video_stream_encoder_->SetSource(&source,
3555 webrtc::DegradationPreference::BALANCED);
3556
3557 int64_t timestamp_ms = kFrameIntervalMs;
3558 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3559 sink_.WaitForEncodedFrame(kWidth, kHeight);
3560 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3561 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3562 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3563 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3564
3565 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3566 video_stream_encoder_->TriggerQualityLow();
3567 timestamp_ms += kFrameIntervalMs;
3568 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3569 sink_.WaitForEncodedFrame(timestamp_ms);
3570 EXPECT_THAT(source.sink_wants(),
3571 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3572 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3573 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3574 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3575
3576 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3577 video_stream_encoder_->TriggerQualityLow();
3578 timestamp_ms += kFrameIntervalMs;
3579 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3580 sink_.WaitForEncodedFrame(timestamp_ms);
3581 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3582 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3583 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3584 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3585
3586 // Trigger adapt down, expect reduced fps (640x360@15fps).
3587 video_stream_encoder_->TriggerQualityLow();
3588 timestamp_ms += kFrameIntervalMs;
3589 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3590 WaitForEncodedFrame(timestamp_ms);
3591 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3592 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3593 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3594 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3595
3596 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3597 source.OnOutputFormatRequest(320, 180);
3598 timestamp_ms += kFrameIntervalMs;
3599 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3600 WaitForEncodedFrame(320, 180);
3601 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3602 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3603
3604 // Trigger adapt down, expect reduced fps (320x180@7fps).
3605 video_stream_encoder_->TriggerCpuOveruse();
3606 timestamp_ms += kFrameIntervalMs;
3607 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3608 WaitForEncodedFrame(timestamp_ms);
3609 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3610 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3611 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3612 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3613 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3614 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3615 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3616
3617 // Source requests HD, expect increased resolution (640x360@7fps).
3618 source.OnOutputFormatRequest(1280, 720);
3619 timestamp_ms += kFrameIntervalMs;
3620 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3621 WaitForEncodedFrame(timestamp_ms);
3622 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3623 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3624
3625 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3626 video_stream_encoder_->TriggerCpuUnderuse();
3627 timestamp_ms += kFrameIntervalMs;
3628 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3629 WaitForEncodedFrame(timestamp_ms);
3630 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3631 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3632 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3633 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3634 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3635 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3636 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3637
3638 // Trigger adapt up, expect increased fps (640x360@maxfps).
3639 video_stream_encoder_->TriggerQualityHigh();
3640 video_stream_encoder_->TriggerCpuUnderuse();
3641 timestamp_ms += kFrameIntervalMs;
3642 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3643 WaitForEncodedFrame(timestamp_ms);
3644 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3645 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3646 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3647 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3648 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3649 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3650 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3651
3652 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3653 video_stream_encoder_->TriggerQualityHigh();
3654 video_stream_encoder_->TriggerCpuUnderuse();
3655 timestamp_ms += kFrameIntervalMs;
3656 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3657 WaitForEncodedFrame(timestamp_ms);
3658 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3659 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3660 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3661 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3662 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3663 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3664 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3665
3666 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3667 video_stream_encoder_->TriggerQualityHigh();
3668 video_stream_encoder_->TriggerCpuUnderuse();
3669 timestamp_ms += kFrameIntervalMs;
3670 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3671 WaitForEncodedFrame(timestamp_ms);
3672 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3673 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3674 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3675 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3676 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3677 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3678 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3679
3680 video_stream_encoder_->Stop();
3681}
3682
3683TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003684 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003685 const int kWidth = 1280;
3686 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003687 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003688 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003689
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003690 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003691 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003692 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003693 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003694
3695 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003696 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003697 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003698 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3699 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3700
3701 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003702 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003703 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003704 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3705 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3706
mflodmancc3d4422017-08-03 08:27:51 -07003707 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003708}
3709
mflodmancc3d4422017-08-03 08:27:51 -07003710TEST_F(VideoStreamEncoderTest,
3711 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003712 const int kWidth = 1280;
3713 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003714 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003715 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003716
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003717 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003718 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003719 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003720 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003721
3722 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003723 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003724 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003725 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003726 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3727
3728 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003729 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003730 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003731 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003732 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3733
mflodmancc3d4422017-08-03 08:27:51 -07003734 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003735}
3736
mflodmancc3d4422017-08-03 08:27:51 -07003737TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003738 const int kWidth = 1280;
3739 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003740 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003741 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003742
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003743 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003744 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003745 video_stream_encoder_->SetSource(&source,
3746 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003747
3748 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3749 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003750 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003751 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3752 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3753 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3754
3755 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003756 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003757 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003758 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3759 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3760 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3761
mflodmancc3d4422017-08-03 08:27:51 -07003762 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003763}
3764
mflodmancc3d4422017-08-03 08:27:51 -07003765TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003766 const int kWidth = 1280;
3767 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003768 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003769 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003770
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003771 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003772 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003773 video_stream_encoder_->SetSource(&source,
3774 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003775
3776 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3777 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003778 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003779 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3780 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3781 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3782
3783 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003784 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003785 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003786 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3787 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3788 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3789
mflodmancc3d4422017-08-03 08:27:51 -07003790 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003791}
3792
mflodmancc3d4422017-08-03 08:27:51 -07003793TEST_F(VideoStreamEncoderTest,
3794 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003795 const int kWidth = 1280;
3796 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003797 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003798 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003799
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003800 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003801 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003802 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003803 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003804 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003805
3806 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003807 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003808 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003809 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3810 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3811
3812 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003813 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003814 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003815 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003816 EXPECT_THAT(source.sink_wants(),
3817 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003818 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3819 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3820
3821 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003822 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003823 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003824 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3825 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3826 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3827
mflodmancc3d4422017-08-03 08:27:51 -07003828 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003829}
3830
mflodmancc3d4422017-08-03 08:27:51 -07003831TEST_F(VideoStreamEncoderTest,
3832 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003833 const int kWidth = 1280;
3834 const int kHeight = 720;
3835 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003836 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003837 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003838
3839 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3840 stats.input_frame_rate = kInputFps;
3841 stats_proxy_->SetMockStats(stats);
3842
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003843 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003844 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3845 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003846 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003847
3848 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003849 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003850 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3851 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003852 EXPECT_THAT(video_source_.sink_wants(),
3853 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003854
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003855 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003856 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003857 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003858 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003859 // Give the encoder queue time to process the change in degradation preference
3860 // by waiting for an encoded frame.
3861 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3862 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003863 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003864
3865 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003866 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003867 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3868 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003869 EXPECT_THAT(new_video_source.sink_wants(),
3870 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003871
3872 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003873 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003874 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003875
mflodmancc3d4422017-08-03 08:27:51 -07003876 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003877}
3878
mflodmancc3d4422017-08-03 08:27:51 -07003879TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003880 const int kWidth = 1280;
3881 const int kHeight = 720;
3882 const size_t kNumFrames = 10;
3883
Henrik Boström381d1092020-05-12 18:49:07 +02003884 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003885 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003886
asaperssond0de2952017-04-21 01:47:31 -07003887 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003888 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003889 video_source_.set_adaptation_enabled(true);
3890
3891 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3892 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3893
3894 int downscales = 0;
3895 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003896 video_source_.IncomingCapturedFrame(
3897 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3898 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003899
asaperssonfab67072017-04-04 05:51:49 -07003900 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003901 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003902 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003903 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003904
3905 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3906 ++downscales;
3907
3908 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3909 EXPECT_EQ(downscales,
3910 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3911 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003912 }
mflodmancc3d4422017-08-03 08:27:51 -07003913 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003914}
3915
mflodmancc3d4422017-08-03 08:27:51 -07003916TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003917 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3918 const int kWidth = 1280;
3919 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003920 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003921 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003922
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003923 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003924 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003925 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003926 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003927 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003928
Åsa Persson8c1bf952018-09-13 10:42:19 +02003929 int64_t timestamp_ms = kFrameIntervalMs;
3930 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003931 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003932 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003933 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3934 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3935
3936 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003937 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003938 timestamp_ms += kFrameIntervalMs;
3939 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3940 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003941 EXPECT_THAT(source.sink_wants(),
3942 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003943 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3944 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3945
3946 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003947 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003948 timestamp_ms += kFrameIntervalMs;
3949 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003950 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003951 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003952 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3953 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3954
3955 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003956 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003957 timestamp_ms += kFrameIntervalMs;
3958 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3959 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003960 EXPECT_THAT(source.sink_wants(),
3961 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003962 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3963 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3964
3965 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003966 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003967 timestamp_ms += kFrameIntervalMs;
3968 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003969 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003970 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003971 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3972 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3973
mflodmancc3d4422017-08-03 08:27:51 -07003974 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003975}
3976
mflodmancc3d4422017-08-03 08:27:51 -07003977TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003978 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3979 const int kWidth = 1280;
3980 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003981 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003982 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003983
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003984 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003985 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003986 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003987 video_stream_encoder_->SetSource(&source,
3988 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003989
Åsa Persson8c1bf952018-09-13 10:42:19 +02003990 int64_t timestamp_ms = kFrameIntervalMs;
3991 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003992 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003993 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003994 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3995 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3996
3997 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003998 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003999 timestamp_ms += kFrameIntervalMs;
4000 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4001 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004002 EXPECT_THAT(source.sink_wants(),
4003 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004004 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4005 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4006
4007 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07004008 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004009 timestamp_ms += kFrameIntervalMs;
4010 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07004011 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004012 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004013 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4014 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4015
4016 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07004017 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004018 timestamp_ms += kFrameIntervalMs;
4019 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4020 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004021 EXPECT_THAT(source.sink_wants(),
4022 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004023 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4024 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4025
4026 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07004027 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004028 timestamp_ms += kFrameIntervalMs;
4029 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07004030 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004031 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004032 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4033 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4034
mflodmancc3d4422017-08-03 08:27:51 -07004035 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004036}
4037
Sergey Silkin41c650b2019-10-14 13:12:19 +02004038TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
4039 fake_encoder_.SetResolutionBitrateLimits(
4040 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4041
Henrik Boström381d1092020-05-12 18:49:07 +02004042 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004043 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4044 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4045 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4046 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004047
4048 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004049 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004050 source.set_adaptation_enabled(true);
4051 video_stream_encoder_->SetSource(
4052 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4053
4054 // Insert 720p frame.
4055 int64_t timestamp_ms = kFrameIntervalMs;
4056 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4057 WaitForEncodedFrame(1280, 720);
4058
4059 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02004060 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004061 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4062 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4063 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4064 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004065 video_stream_encoder_->TriggerQualityLow();
4066
4067 // Insert 720p frame. It should be downscaled and encoded.
4068 timestamp_ms += kFrameIntervalMs;
4069 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4070 WaitForEncodedFrame(960, 540);
4071
4072 // Trigger adapt up. Higher resolution should not be requested duo to lack
4073 // of bitrate.
4074 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004075 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02004076
4077 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02004078 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004079 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4080 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4081 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4082 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004083
4084 // Trigger adapt up. Higher resolution should be requested.
4085 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004086 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02004087
4088 video_stream_encoder_->Stop();
4089}
4090
4091TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
4092 fake_encoder_.SetResolutionBitrateLimits(
4093 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4094
4095 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02004096 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004097 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4098 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4099 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4100 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004101
4102 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004103 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004104 source.set_adaptation_enabled(true);
4105 video_stream_encoder_->SetSource(
4106 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4107
4108 // Insert 720p frame. It should be dropped and lower resolution should be
4109 // requested.
4110 int64_t timestamp_ms = kFrameIntervalMs;
4111 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4112 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004113 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004114
4115 // Insert 720p frame. It should be downscaled and encoded.
4116 timestamp_ms += kFrameIntervalMs;
4117 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4118 WaitForEncodedFrame(960, 540);
4119
4120 video_stream_encoder_->Stop();
4121}
4122
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004123class BalancedDegradationTest : public VideoStreamEncoderTest {
4124 protected:
4125 void SetupTest() {
4126 // Reset encoder for field trials to take effect.
4127 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02004128 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004129
4130 // Enable BALANCED preference.
4131 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02004132 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4133 }
4134
Asa Persson606d3cb2021-10-04 10:07:11 +02004135 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02004136 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004137 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004138 }
4139
Åsa Persson45b176f2019-09-30 11:19:05 +02004140 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004141 timestamp_ms_ += kFrameIntervalMs;
4142 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02004143 }
4144
4145 void InsertFrameAndWaitForEncoded() {
4146 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004147 sink_.WaitForEncodedFrame(timestamp_ms_);
4148 }
4149
4150 const int kWidth = 640; // pixels:640x360=230400
4151 const int kHeight = 360;
4152 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4153 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004154 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004155};
4156
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004157TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004158 test::ScopedKeyValueConfig field_trials(
4159 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004160 "WebRTC-Video-BalancedDegradationSettings/"
4161 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4162 SetupTest();
4163
4164 // Force input frame rate.
4165 const int kInputFps = 24;
4166 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4167 stats.input_frame_rate = kInputFps;
4168 stats_proxy_->SetMockStats(stats);
4169
Åsa Persson45b176f2019-09-30 11:19:05 +02004170 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004171 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004172
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004173 // Trigger adapt down, expect scaled down framerate and resolution,
4174 // since Fps diff (input-requested:0) < threshold.
4175 video_stream_encoder_->TriggerQualityLow();
4176 EXPECT_THAT(source_.sink_wants(),
4177 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004178
4179 video_stream_encoder_->Stop();
4180}
4181
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004182TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004183 test::ScopedKeyValueConfig field_trials(
4184 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004185 "WebRTC-Video-BalancedDegradationSettings/"
4186 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4187 SetupTest();
4188
4189 // Force input frame rate.
4190 const int kInputFps = 25;
4191 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4192 stats.input_frame_rate = kInputFps;
4193 stats_proxy_->SetMockStats(stats);
4194
Åsa Persson45b176f2019-09-30 11:19:05 +02004195 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004196 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004197
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004198 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4199 // Fps diff (input-requested:1) == threshold.
4200 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004201 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004202
4203 video_stream_encoder_->Stop();
4204}
4205
4206TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004207 test::ScopedKeyValueConfig field_trials(
4208 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004209 "WebRTC-Video-BalancedDegradationSettings/"
4210 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4211 SetupTest();
4212
4213 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4214
Åsa Persson45b176f2019-09-30 11:19:05 +02004215 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004216 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004217
4218 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4219 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004220 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004221
4222 video_stream_encoder_->Stop();
4223}
4224
Åsa Perssonccfb3402019-09-25 15:13:04 +02004225TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004226 test::ScopedKeyValueConfig field_trials(
4227 field_trials_,
Åsa Persson1b247f12019-08-14 17:26:39 +02004228 "WebRTC-Video-BalancedDegradationSettings/"
4229 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004230 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004231
Asa Persson606d3cb2021-10-04 10:07:11 +02004232 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4233 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4234 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004235
Åsa Persson45b176f2019-09-30 11:19:05 +02004236 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004237 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004238 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4239
4240 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4241 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004242 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004243 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004244 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4245
4246 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4247 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004248 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004249 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004250 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4251
Åsa Persson30ab0152019-08-27 12:22:33 +02004252 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4253 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004254 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004255 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004256 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004257 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4258
4259 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004260 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004261 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004262 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004263
Åsa Persson30ab0152019-08-27 12:22:33 +02004264 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004265 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004266 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004267 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004268 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004269 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4270
4271 video_stream_encoder_->Stop();
4272}
4273
Åsa Perssonccfb3402019-09-25 15:13:04 +02004274TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004275 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004276 test::ScopedKeyValueConfig field_trials(
4277 field_trials_,
Åsa Persson45b176f2019-09-30 11:19:05 +02004278 "WebRTC-Video-BalancedDegradationSettings/"
4279 "pixels:57600|129600|230400,fps:7|24|24/");
4280 SetupTest();
Asa Persson606d3cb2021-10-04 10:07:11 +02004281 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 11:19:05 +02004282
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004283 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004284
4285 // Insert frame, expect scaled down:
4286 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4287 InsertFrame();
4288 EXPECT_FALSE(WaitForFrame(1000));
4289 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4290 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4291
4292 // Insert frame, expect scaled down:
4293 // resolution (320x180@24fps).
4294 InsertFrame();
4295 EXPECT_FALSE(WaitForFrame(1000));
4296 EXPECT_LT(source_.sink_wants().max_pixel_count,
4297 source_.last_wants().max_pixel_count);
4298 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4299
4300 // Frame should not be dropped (min pixels per frame reached).
4301 InsertFrameAndWaitForEncoded();
4302
4303 video_stream_encoder_->Stop();
4304}
4305
4306TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004307 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004308 test::ScopedKeyValueConfig field_trials(
4309 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004310 "WebRTC-Video-BalancedDegradationSettings/"
4311 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004312 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004313
Asa Persson606d3cb2021-10-04 10:07:11 +02004314 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4315 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4316 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004317
Åsa Persson45b176f2019-09-30 11:19:05 +02004318 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004319 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004320 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4321
4322 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4323 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004324 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004325 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004326 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4327
4328 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4329 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004330 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004331 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004332 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4333
4334 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4335 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004336 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004337 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004338 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4339
Åsa Persson30ab0152019-08-27 12:22:33 +02004340 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4341 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004342 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004343 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004344 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4345
4346 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4347 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004348 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004349 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4350
4351 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004352 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004353 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004354 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004355 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004356 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4357
4358 video_stream_encoder_->Stop();
4359}
4360
Åsa Perssonccfb3402019-09-25 15:13:04 +02004361TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004362 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004363 test::ScopedKeyValueConfig field_trials(
4364 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004365 "WebRTC-Video-BalancedDegradationSettings/"
4366 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004367 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004368
Asa Persson606d3cb2021-10-04 10:07:11 +02004369 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4370 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4371 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4372 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4373 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004374
Åsa Persson45b176f2019-09-30 11:19:05 +02004375 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004376 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004377 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4378
4379 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4380 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004381 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004382 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004383 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4384
4385 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4386 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004387 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004388 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004389 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4390
4391 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4392 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004393 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004394 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004395 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4396
4397 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4398 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004399 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004400 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4401
4402 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004403 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004404 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004405 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004406 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004407 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4408
4409 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004410 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004411 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004412 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004413 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4414
4415 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004416 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004417 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004418 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004419 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004420 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4421
Åsa Persson1b247f12019-08-14 17:26:39 +02004422 video_stream_encoder_->Stop();
4423}
4424
mflodmancc3d4422017-08-03 08:27:51 -07004425TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004426 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4427 const int kWidth = 1280;
4428 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004429 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004430 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004431
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004432 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004433 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004434 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004435 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004436 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004437
Åsa Persson8c1bf952018-09-13 10:42:19 +02004438 int64_t timestamp_ms = kFrameIntervalMs;
4439 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004440 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004441 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004442 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4443 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4444 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4445 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4446
4447 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004448 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004449 timestamp_ms += kFrameIntervalMs;
4450 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4451 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004452 EXPECT_THAT(source.sink_wants(),
4453 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004454 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4455 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4456 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4457 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4458
4459 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004460 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004461 timestamp_ms += kFrameIntervalMs;
4462 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4463 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004464 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004465 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4466 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4467 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4468 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4469
Jonathan Yubc771b72017-12-08 17:04:29 -08004470 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004471 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004472 timestamp_ms += kFrameIntervalMs;
4473 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4474 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004475 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004476 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4477 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004478 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004479 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4480
Jonathan Yubc771b72017-12-08 17:04:29 -08004481 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004482 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004483 timestamp_ms += kFrameIntervalMs;
4484 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4485 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004486 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004487 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004488 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4489 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4490 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4491 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4492
Jonathan Yubc771b72017-12-08 17:04:29 -08004493 // Trigger quality adapt down, expect no change (min resolution reached).
4494 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004495 timestamp_ms += kFrameIntervalMs;
4496 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4497 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004498 EXPECT_THAT(source.sink_wants(), FpsMax());
4499 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004500 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4501 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4502 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4503 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4504
Evan Shrubsole64469032020-06-11 10:45:29 +02004505 // Trigger quality adapt up, expect upscaled resolution (480x270).
4506 video_stream_encoder_->TriggerQualityHigh();
4507 timestamp_ms += kFrameIntervalMs;
4508 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4509 WaitForEncodedFrame(timestamp_ms);
4510 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4511 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4512 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4513 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4514 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4515
4516 // Trigger quality and cpu adapt up since both are most limited, expect
4517 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004518 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004519 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004520 timestamp_ms += kFrameIntervalMs;
4521 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4522 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004523 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004524 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4525 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4526 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004527 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004528
Evan Shrubsole64469032020-06-11 10:45:29 +02004529 // Trigger quality and cpu adapt up since both are most limited, expect
4530 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004531 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004532 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004533 timestamp_ms += kFrameIntervalMs;
4534 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4535 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004536 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004537 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004538 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004539 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004540 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4541 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004542
Evan Shrubsole64469032020-06-11 10:45:29 +02004543 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4544 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004545 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004546 timestamp_ms += kFrameIntervalMs;
4547 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4548 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004549 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004550 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4551 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004552 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004553 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004554
4555 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004556 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004557 timestamp_ms += kFrameIntervalMs;
4558 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004559 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004560 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004561 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004562 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4563 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004564 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004565 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004566
mflodmancc3d4422017-08-03 08:27:51 -07004567 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004568}
4569
mflodmancc3d4422017-08-03 08:27:51 -07004570TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004571 const int kWidth = 640;
4572 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004573
Henrik Boström381d1092020-05-12 18:49:07 +02004574 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004575 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004576
perkj803d97f2016-11-01 11:45:46 -07004577 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004578 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004579 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004580 }
4581
mflodmancc3d4422017-08-03 08:27:51 -07004582 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004583 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004584 video_source_.IncomingCapturedFrame(CreateFrame(
4585 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004586 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004587 }
4588
mflodmancc3d4422017-08-03 08:27:51 -07004589 video_stream_encoder_->Stop();
4590 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004591 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004592
Ying Wangef3998f2019-12-09 13:06:53 +01004593 EXPECT_METRIC_EQ(
4594 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4595 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004596 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4597}
4598
mflodmancc3d4422017-08-03 08:27:51 -07004599TEST_F(VideoStreamEncoderTest,
4600 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004601 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004602 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004603 const int kWidth = 640;
4604 const int kHeight = 360;
4605
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004606 video_stream_encoder_->SetSource(&video_source_,
4607 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004608
4609 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4610 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004611 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004612 }
4613
mflodmancc3d4422017-08-03 08:27:51 -07004614 video_stream_encoder_->Stop();
4615 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004616 stats_proxy_.reset();
4617
4618 EXPECT_EQ(0,
4619 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4620}
4621
Per Kjellanderdcef6412020-10-07 15:09:05 +02004622TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4623 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004624 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004625 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004626
4627 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004628 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 10:07:11 +02004629 SimulcastRateAllocator(fake_encoder_.config())
4630 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004631 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004632
Henrik Boström381d1092020-05-12 18:49:07 +02004633 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004634 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004635
sprang57c2fff2017-01-16 06:24:02 -08004636 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004637 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4638 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004639 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4640 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4641
Erik Språngd7329ca2019-02-21 21:19:53 +01004642 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004643 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004644 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004645
Per Kjellanderdcef6412020-10-07 15:09:05 +02004646 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004647 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004648 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4649 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004650 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004651 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004652
Per Kjellanderdcef6412020-10-07 15:09:05 +02004653 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004654 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004655 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004656 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004657 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4658 WaitForEncodedFrame(CurrentTimeMs());
4659 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004660 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004661 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004662
mflodmancc3d4422017-08-03 08:27:51 -07004663 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004664}
4665
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004666TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004667 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004668 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004669 kVideoLayersAllocation);
4670
4671 const int kDefaultFps = 30;
4672
4673 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004674 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004675
4676 video_source_.IncomingCapturedFrame(
4677 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4678 WaitForEncodedFrame(CurrentTimeMs());
4679 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4680 VideoLayersAllocation last_layer_allocation =
4681 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004682 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004683 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4684
4685 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004686 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004687 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 10:07:11 +02004688 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 17:53:22 +02004689 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4690
Erik Språng9d69cbe2020-10-22 17:44:42 +02004691 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004692 int number_of_layers_allocation = 1;
4693 const int64_t start_time_ms = CurrentTimeMs();
4694 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4695 video_source_.IncomingCapturedFrame(
4696 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4697 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004698 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4699 number_of_layers_allocation = sink_.number_of_layers_allocations();
4700 VideoLayersAllocation new_allocation =
4701 sink_.GetLastVideoLayersAllocation();
4702 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4703 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4704 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4705 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4706 .target_bitrate_per_temporal_layer,
4707 last_layer_allocation.active_spatial_layers[0]
4708 .target_bitrate_per_temporal_layer);
4709 last_layer_allocation = new_allocation;
4710 }
4711 }
4712 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4713 video_stream_encoder_->Stop();
4714}
4715
4716TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004717 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004718 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4719 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4720 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004721 VideoEncoderConfig video_encoder_config;
4722 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4723 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004724 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004725 video_encoder_config.content_type =
4726 VideoEncoderConfig::ContentType::kRealtimeVideo;
4727 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004728 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004729 VideoEncoder::GetDefaultVp8Settings());
4730 for (auto& layer : video_encoder_config.simulcast_layers) {
4731 layer.num_temporal_layers = 2;
4732 }
4733 // Simulcast layers are used for enabling/disabling streams.
4734 video_encoder_config.simulcast_layers[0].active = true;
4735 video_encoder_config.simulcast_layers[1].active = false;
4736 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004737 ConfigureEncoder(std::move(video_encoder_config),
4738 VideoStreamEncoder::BitrateAllocationCallbackType::
4739 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004740
4741 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004742 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004743
4744 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4745 WaitForEncodedFrame(CurrentTimeMs());
4746 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4747 VideoLayersAllocation last_layer_allocation =
4748 sink_.GetLastVideoLayersAllocation();
4749
4750 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4751 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4752 .target_bitrate_per_temporal_layer,
4753 SizeIs(2));
4754 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4755 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4756 video_stream_encoder_->Stop();
4757}
4758
4759TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004760 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004761 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4762 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4763 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004764 VideoEncoderConfig video_encoder_config;
4765 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4766 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004767 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004768 video_encoder_config.content_type =
4769 VideoEncoderConfig::ContentType::kRealtimeVideo;
4770 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004771 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004772 VideoEncoder::GetDefaultVp8Settings());
4773 for (auto& layer : video_encoder_config.simulcast_layers) {
4774 layer.num_temporal_layers = 2;
4775 }
4776 // Simulcast layers are used for enabling/disabling streams.
4777 video_encoder_config.simulcast_layers[0].active = true;
4778 video_encoder_config.simulcast_layers[1].active = false;
4779 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004780 ConfigureEncoder(std::move(video_encoder_config),
4781 VideoStreamEncoder::BitrateAllocationCallbackType::
4782 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004783
4784 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004785 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004786
4787 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4788 WaitForEncodedFrame(CurrentTimeMs());
4789 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4790 VideoLayersAllocation last_layer_allocation =
4791 sink_.GetLastVideoLayersAllocation();
4792
4793 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4794 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4795 .target_bitrate_per_temporal_layer,
4796 SizeIs(2));
4797 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4798
4799 video_stream_encoder_->Stop();
4800}
4801
4802TEST_F(VideoStreamEncoderTest,
4803 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4804 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4805 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004806 VideoEncoderConfig video_encoder_config;
4807 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4808 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004809 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004810 video_encoder_config.content_type =
4811 VideoEncoderConfig::ContentType::kRealtimeVideo;
4812 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4813 vp9_settings.numberOfSpatialLayers = 2;
4814 vp9_settings.numberOfTemporalLayers = 2;
4815 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4816 vp9_settings.automaticResizeOn = false;
4817 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004818 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004819 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004820 ConfigureEncoder(std::move(video_encoder_config),
4821 VideoStreamEncoder::BitrateAllocationCallbackType::
4822 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004823
4824 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004825 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004826
4827 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4828 WaitForEncodedFrame(CurrentTimeMs());
4829 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4830 VideoLayersAllocation last_layer_allocation =
4831 sink_.GetLastVideoLayersAllocation();
4832
4833 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4834 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4835 .target_bitrate_per_temporal_layer,
4836 SizeIs(2));
4837 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4838 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4839 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4840 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4841 .target_bitrate_per_temporal_layer,
4842 SizeIs(2));
4843 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4844 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4845 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4846
4847 // Since full SVC is used, expect the top layer to utilize the full target
4848 // rate.
4849 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4850 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004851 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004852 video_stream_encoder_->Stop();
4853}
4854
4855TEST_F(VideoStreamEncoderTest,
4856 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4857 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4858 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004859 VideoEncoderConfig video_encoder_config;
4860 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4861 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004862 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004863 video_encoder_config.content_type =
4864 VideoEncoderConfig::ContentType::kRealtimeVideo;
4865 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4866 vp9_settings.numberOfSpatialLayers = 2;
4867 vp9_settings.numberOfTemporalLayers = 2;
4868 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4869 vp9_settings.automaticResizeOn = false;
4870 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004871 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004872 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004873 ConfigureEncoder(std::move(video_encoder_config),
4874 VideoStreamEncoder::BitrateAllocationCallbackType::
4875 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004876
4877 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004878 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004879
4880 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4881 WaitForEncodedFrame(CurrentTimeMs());
4882 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4883 VideoLayersAllocation last_layer_allocation =
4884 sink_.GetLastVideoLayersAllocation();
4885
4886 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4887 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4888 .target_bitrate_per_temporal_layer,
4889 SizeIs(1));
4890 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4891 .target_bitrate_per_temporal_layer,
4892 SizeIs(1));
4893 // Since full SVC is used, expect the top layer to utilize the full target
4894 // rate.
4895 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4896 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004897 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004898 video_stream_encoder_->Stop();
4899}
4900
4901TEST_F(VideoStreamEncoderTest,
4902 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4903 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4904 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004905 VideoEncoderConfig video_encoder_config;
4906 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4907 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004908 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004909 video_encoder_config.content_type =
4910 VideoEncoderConfig::ContentType::kRealtimeVideo;
4911 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4912 vp9_settings.numberOfSpatialLayers = 2;
4913 vp9_settings.numberOfTemporalLayers = 2;
4914 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4915 vp9_settings.automaticResizeOn = false;
4916 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004917 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004918 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004919 ConfigureEncoder(std::move(video_encoder_config),
4920 VideoStreamEncoder::BitrateAllocationCallbackType::
4921 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004922
4923 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004924 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004925
4926 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4927 WaitForEncodedFrame(CurrentTimeMs());
4928 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4929 VideoLayersAllocation last_layer_allocation =
4930 sink_.GetLastVideoLayersAllocation();
4931
4932 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4933 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4934 .target_bitrate_per_temporal_layer,
4935 SizeIs(2));
4936 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4937 .target_bitrate_per_temporal_layer,
4938 SizeIs(2));
4939 // Since KSVC is, spatial layers are independend except on key frames.
4940 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4941 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004942 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004943 video_stream_encoder_->Stop();
4944}
4945
4946TEST_F(VideoStreamEncoderTest,
4947 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4948 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4949 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4950 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004951 VideoEncoderConfig video_encoder_config;
4952 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4953 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004954 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004955 video_encoder_config.content_type =
4956 VideoEncoderConfig::ContentType::kRealtimeVideo;
4957 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4958 vp9_settings.numberOfSpatialLayers = 3;
4959 vp9_settings.numberOfTemporalLayers = 2;
4960 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4961 vp9_settings.automaticResizeOn = false;
4962 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004963 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004964 vp9_settings);
4965 // Simulcast layers are used for enabling/disabling streams.
4966 video_encoder_config.simulcast_layers.resize(3);
4967 video_encoder_config.simulcast_layers[0].active = false;
4968 video_encoder_config.simulcast_layers[1].active = true;
4969 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004970 ConfigureEncoder(std::move(video_encoder_config),
4971 VideoStreamEncoder::BitrateAllocationCallbackType::
4972 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004973
4974 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004975 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004976
4977 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4978 WaitForEncodedFrame(CurrentTimeMs());
4979 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4980 VideoLayersAllocation last_layer_allocation =
4981 sink_.GetLastVideoLayersAllocation();
4982
4983 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4984 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4985 .target_bitrate_per_temporal_layer,
4986 SizeIs(2));
4987 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4988 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4989
4990 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4991 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4992 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4993 .target_bitrate_per_temporal_layer,
4994 SizeIs(2));
4995 // Since full SVC is used, expect the top layer to utilize the full target
4996 // rate.
4997 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4998 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004999 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005000 video_stream_encoder_->Stop();
5001}
5002
5003TEST_F(VideoStreamEncoderTest,
5004 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
5005 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5006 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5007 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005008 VideoEncoderConfig video_encoder_config;
5009 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5010 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005011 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005012 video_encoder_config.content_type =
5013 VideoEncoderConfig::ContentType::kRealtimeVideo;
5014 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5015 vp9_settings.numberOfSpatialLayers = 3;
5016 vp9_settings.numberOfTemporalLayers = 2;
5017 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5018 vp9_settings.automaticResizeOn = false;
5019 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005020 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005021 vp9_settings);
5022 // Simulcast layers are used for enabling/disabling streams.
5023 video_encoder_config.simulcast_layers.resize(3);
5024 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005025 ConfigureEncoder(std::move(video_encoder_config),
5026 VideoStreamEncoder::BitrateAllocationCallbackType::
5027 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005028
5029 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005030 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005031
5032 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5033 WaitForEncodedFrame(CurrentTimeMs());
5034 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5035 VideoLayersAllocation last_layer_allocation =
5036 sink_.GetLastVideoLayersAllocation();
5037
5038 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
5039 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5040 .target_bitrate_per_temporal_layer,
5041 SizeIs(2));
5042 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
5043 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5044
5045 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
5046 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
5047 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
5048 .target_bitrate_per_temporal_layer,
5049 SizeIs(2));
5050 video_stream_encoder_->Stop();
5051}
5052
5053TEST_F(VideoStreamEncoderTest,
5054 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
5055 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5056 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5057 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005058 VideoEncoderConfig video_encoder_config;
5059 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5060 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005061 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005062 video_encoder_config.content_type =
5063 VideoEncoderConfig::ContentType::kRealtimeVideo;
5064 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5065 vp9_settings.numberOfSpatialLayers = 3;
5066 vp9_settings.numberOfTemporalLayers = 2;
5067 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5068 vp9_settings.automaticResizeOn = false;
5069 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005070 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005071 vp9_settings);
5072 // Simulcast layers are used for enabling/disabling streams.
5073 video_encoder_config.simulcast_layers.resize(3);
5074 video_encoder_config.simulcast_layers[0].active = false;
5075 video_encoder_config.simulcast_layers[1].active = false;
5076 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005077 ConfigureEncoder(std::move(video_encoder_config),
5078 VideoStreamEncoder::BitrateAllocationCallbackType::
5079 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005080
5081 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005082 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005083
5084 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5085 WaitForEncodedFrame(CurrentTimeMs());
5086 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5087 VideoLayersAllocation last_layer_allocation =
5088 sink_.GetLastVideoLayersAllocation();
5089
5090 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5091 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5092 .target_bitrate_per_temporal_layer,
5093 SizeIs(2));
5094 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5095 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5096 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5097 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02005098 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005099 video_stream_encoder_->Stop();
5100}
5101
5102TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5103 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005104 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005105 kVideoLayersAllocation);
5106 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005107 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005108
5109 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5110 WaitForEncodedFrame(CurrentTimeMs());
5111 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5112 VideoLayersAllocation last_layer_allocation =
5113 sink_.GetLastVideoLayersAllocation();
5114
5115 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5116 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5117 .target_bitrate_per_temporal_layer,
5118 SizeIs(1));
5119 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5120 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005121 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005122 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5123 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5124 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5125 video_stream_encoder_->Stop();
5126}
5127
5128TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02005129 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5130 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005131 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02005132 kVideoLayersAllocation);
5133
5134 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005135 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005136
5137 video_source_.IncomingCapturedFrame(
5138 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5139 WaitForEncodedFrame(CurrentTimeMs());
5140 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5141 VideoLayersAllocation last_layer_allocation =
5142 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02005143 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02005144 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5145 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5146 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005147 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 17:53:22 +02005148
5149 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005150 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5151 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005152 video_source_.IncomingCapturedFrame(
5153 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5154 WaitForEncodedFrame(CurrentTimeMs());
5155
5156 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5157 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5158 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5159 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5160 .target_bitrate_per_temporal_layer[0],
5161 DataRate::Zero());
5162
5163 video_stream_encoder_->Stop();
5164}
5165
Per Kjellander4190ce92020-12-15 17:24:55 +01005166TEST_F(VideoStreamEncoderTest,
5167 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5168 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005169 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01005170 kVideoLayersAllocation);
5171
5172 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005173 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5174 0, 0, 0);
Per Kjellander4190ce92020-12-15 17:24:55 +01005175
5176 video_source_.IncomingCapturedFrame(
5177 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5178 WaitForEncodedFrame(CurrentTimeMs());
5179 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5180 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5181 SizeIs(2));
5182 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5183 codec_width_);
5184 EXPECT_EQ(
5185 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5186 codec_height_);
5187
5188 video_source_.IncomingCapturedFrame(
5189 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5190 WaitForEncodedFrame(CurrentTimeMs());
5191 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5192 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5193 SizeIs(2));
5194 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5195 codec_width_ / 2);
5196 EXPECT_EQ(
5197 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5198 codec_height_ / 2);
5199
5200 video_stream_encoder_->Stop();
5201}
5202
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005203TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5204 // 2 TLs configured, temporal layers supported by encoder.
5205 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005206 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005207 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005208 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005209 fake_encoder_.SetTemporalLayersSupported(0, true);
5210
5211 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 10:07:11 +02005212 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005213 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005214 kNumTemporalLayers, /*temporal_id*/ 0,
5215 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005216 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005217 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005218 kNumTemporalLayers, /*temporal_id*/ 1,
5219 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005220 VideoBitrateAllocation expected_bitrate;
5221 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5222 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5223
5224 VerifyAllocatedBitrate(expected_bitrate);
5225 video_stream_encoder_->Stop();
5226}
5227
5228TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5229 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005230 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005231 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005232 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005233 fake_encoder_.SetTemporalLayersSupported(0, false);
5234
5235 // Temporal layers not supported by the encoder.
5236 // Total bitrate should be at ti:0.
5237 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005238 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005239
5240 VerifyAllocatedBitrate(expected_bitrate);
5241 video_stream_encoder_->Stop();
5242}
5243
5244TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005245 webrtc::test::ScopedKeyValueConfig field_trials(
5246 field_trials_,
Per Kjellanderdcef6412020-10-07 15:09:05 +02005247 "WebRTC-Video-QualityScalerSettings/"
5248 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5249 // Reset encoder for field trials to take effect.
5250 ConfigureEncoder(video_encoder_config_.Copy());
5251
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005252 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005253 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005254 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005255 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005256 fake_encoder_.SetTemporalLayersSupported(0, true);
5257 fake_encoder_.SetTemporalLayersSupported(1, false);
5258
5259 const int kS0Bps = 150000;
5260 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005261 kS0Bps *
5262 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5263 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005264 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005265 kS0Bps *
5266 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5267 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005268 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005269 // Temporal layers not supported by si:1.
5270 VideoBitrateAllocation expected_bitrate;
5271 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5272 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5273 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5274
5275 VerifyAllocatedBitrate(expected_bitrate);
5276 video_stream_encoder_->Stop();
5277}
5278
Niels Möller7dc26b72017-12-06 10:27:48 +01005279TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5280 const int kFrameWidth = 1280;
5281 const int kFrameHeight = 720;
5282 const int kFramerate = 24;
5283
Henrik Boström381d1092020-05-12 18:49:07 +02005284 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005285 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005286 test::FrameForwarder source;
5287 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005288 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005289
5290 // Insert a single frame, triggering initial configuration.
5291 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5292 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5293
5294 EXPECT_EQ(
5295 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5296 kDefaultFramerate);
5297
5298 // Trigger reconfigure encoder (without resetting the entire instance).
5299 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005300 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5301 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005302 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005303 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005304 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005305 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5306
5307 // Detector should be updated with fps limit from codec config.
5308 EXPECT_EQ(
5309 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5310 kFramerate);
5311
5312 // Trigger overuse, max framerate should be reduced.
5313 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5314 stats.input_frame_rate = kFramerate;
5315 stats_proxy_->SetMockStats(stats);
5316 video_stream_encoder_->TriggerCpuOveruse();
5317 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5318 int adapted_framerate =
5319 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5320 EXPECT_LT(adapted_framerate, kFramerate);
5321
5322 // Trigger underuse, max framerate should go back to codec configured fps.
5323 // Set extra low fps, to make sure it's actually reset, not just incremented.
5324 stats = stats_proxy_->GetStats();
5325 stats.input_frame_rate = adapted_framerate / 2;
5326 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005327 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005328 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5329 EXPECT_EQ(
5330 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5331 kFramerate);
5332
5333 video_stream_encoder_->Stop();
5334}
5335
5336TEST_F(VideoStreamEncoderTest,
5337 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5338 const int kFrameWidth = 1280;
5339 const int kFrameHeight = 720;
5340 const int kLowFramerate = 15;
5341 const int kHighFramerate = 25;
5342
Henrik Boström381d1092020-05-12 18:49:07 +02005343 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005344 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005345 test::FrameForwarder source;
5346 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005347 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005348
5349 // Trigger initial configuration.
5350 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005351 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5352 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005353 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005354 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005355 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005356 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005357 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5358
5359 EXPECT_EQ(
5360 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5361 kLowFramerate);
5362
5363 // Trigger overuse, max framerate should be reduced.
5364 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5365 stats.input_frame_rate = kLowFramerate;
5366 stats_proxy_->SetMockStats(stats);
5367 video_stream_encoder_->TriggerCpuOveruse();
5368 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5369 int adapted_framerate =
5370 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5371 EXPECT_LT(adapted_framerate, kLowFramerate);
5372
5373 // Reconfigure the encoder with a new (higher max framerate), max fps should
5374 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005375 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005376 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5377 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005378 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005379 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5380
5381 EXPECT_EQ(
5382 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5383 adapted_framerate);
5384
5385 // Trigger underuse, max framerate should go back to codec configured fps.
5386 stats = stats_proxy_->GetStats();
5387 stats.input_frame_rate = adapted_framerate;
5388 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005389 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005390 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5391 EXPECT_EQ(
5392 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5393 kHighFramerate);
5394
5395 video_stream_encoder_->Stop();
5396}
5397
mflodmancc3d4422017-08-03 08:27:51 -07005398TEST_F(VideoStreamEncoderTest,
5399 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005400 const int kFrameWidth = 1280;
5401 const int kFrameHeight = 720;
5402 const int kFramerate = 24;
5403
Henrik Boström381d1092020-05-12 18:49:07 +02005404 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005405 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005406 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005407 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005408 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005409
5410 // Trigger initial configuration.
5411 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005412 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5413 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005414 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 04:21:07 -07005415 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005416 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005417 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005418 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005419
Niels Möller7dc26b72017-12-06 10:27:48 +01005420 EXPECT_EQ(
5421 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5422 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005423
5424 // Trigger overuse, max framerate should be reduced.
5425 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5426 stats.input_frame_rate = kFramerate;
5427 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005428 video_stream_encoder_->TriggerCpuOveruse();
5429 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005430 int adapted_framerate =
5431 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005432 EXPECT_LT(adapted_framerate, kFramerate);
5433
5434 // Change degradation preference to not enable framerate scaling. Target
5435 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005436 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005437 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005438 EXPECT_EQ(
5439 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5440 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005441
mflodmancc3d4422017-08-03 08:27:51 -07005442 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005443}
5444
mflodmancc3d4422017-08-03 08:27:51 -07005445TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005446 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005447 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005448 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5449 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5450 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005451 const int kWidth = 640;
5452 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005453
asaperssonfab67072017-04-04 05:51:49 -07005454 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005455
5456 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005457 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005458
5459 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005460 EXPECT_TRUE_WAIT(
5461 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005462
sprangc5d62e22017-04-02 23:53:04 -07005463 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005464
asaperssonfab67072017-04-04 05:51:49 -07005465 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005466 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005467 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005468
5469 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005470 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005471
Henrik Boström2671dac2020-05-19 16:29:09 +02005472 EXPECT_TRUE_WAIT(
5473 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005474
mflodmancc3d4422017-08-03 08:27:51 -07005475 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005476}
5477
mflodmancc3d4422017-08-03 08:27:51 -07005478TEST_F(VideoStreamEncoderTest,
5479 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005480 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005481 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005482 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5483 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5484 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005485 const int kWidth = 640;
5486 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005487
5488 // We expect the n initial frames to get dropped.
5489 int i;
5490 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005491 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005492 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005493 }
5494 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005495 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005496 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005497
5498 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005499 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005500
mflodmancc3d4422017-08-03 08:27:51 -07005501 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005502}
5503
mflodmancc3d4422017-08-03 08:27:51 -07005504TEST_F(VideoStreamEncoderTest,
5505 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005506 const int kWidth = 640;
5507 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005508 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005509 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005510
5511 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005512 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005513 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005514
asaperssonfab67072017-04-04 05:51:49 -07005515 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005516 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005517 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005518
mflodmancc3d4422017-08-03 08:27:51 -07005519 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005520}
5521
mflodmancc3d4422017-08-03 08:27:51 -07005522TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005523 const int kWidth = 640;
5524 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005525 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005526
5527 VideoEncoderConfig video_encoder_config;
5528 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5529 // Make format different, to force recreation of encoder.
5530 video_encoder_config.video_format.parameters["foo"] = "foo";
5531 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005532 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005533 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005534 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005535
kthelgasonb83797b2017-02-14 11:57:25 -08005536 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005537 video_stream_encoder_->SetSource(&video_source_,
5538 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005539
asaperssonfab67072017-04-04 05:51:49 -07005540 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005541 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005542 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005543
mflodmancc3d4422017-08-03 08:27:51 -07005544 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005545 fake_encoder_.SetQualityScaling(true);
5546}
5547
Åsa Persson139f4dc2019-08-02 09:29:58 +02005548TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005549 webrtc::test::ScopedKeyValueConfig field_trials(
5550 field_trials_,
Åsa Persson139f4dc2019-08-02 09:29:58 +02005551 "WebRTC-Video-QualityScalerSettings/"
5552 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5553 // Reset encoder for field trials to take effect.
5554 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005555 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5556 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 09:29:58 +02005557 const int kWidth = 640;
5558 const int kHeight = 360;
5559
Henrik Boström381d1092020-05-12 18:49:07 +02005560 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005561 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005562 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5563 // Frame should not be dropped.
5564 WaitForEncodedFrame(1);
5565
Henrik Boström381d1092020-05-12 18:49:07 +02005566 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005567 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5568 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5569 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005570 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5571 // Frame should not be dropped.
5572 WaitForEncodedFrame(2);
5573
Henrik Boström381d1092020-05-12 18:49:07 +02005574 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005575 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5576 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5577 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005578 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5579 // Expect to drop this frame, the wait should time out.
5580 ExpectDroppedFrame();
5581
5582 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005583 EXPECT_TRUE_WAIT(
5584 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005585 video_stream_encoder_->Stop();
5586}
5587
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005588TEST_F(VideoStreamEncoderTest,
5589 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005590 webrtc::test::ScopedKeyValueConfig field_trials(
5591 field_trials_,
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005592 "WebRTC-Video-QualityScalerSettings/"
5593 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5594 fake_encoder_.SetQualityScaling(false);
5595 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005596 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5597 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005598 const int kWidth = 640;
5599 const int kHeight = 360;
5600
5601 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005602 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005603 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5604 // Frame should not be dropped.
5605 WaitForEncodedFrame(1);
5606
5607 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5608 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5609 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5610 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5611 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5612 // Frame should not be dropped.
5613 WaitForEncodedFrame(2);
5614
5615 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5616 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5617 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5618 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5619 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5620 // Not dropped since quality scaling is disabled.
5621 WaitForEncodedFrame(3);
5622
5623 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005624 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005625 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5626
5627 video_stream_encoder_->Stop();
5628}
5629
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005630TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005631 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005632 // Set simulcast.
5633 ResetEncoder("VP8", 3, 1, 1, false);
5634 fake_encoder_.SetQualityScaling(true);
5635 const int kWidth = 1280;
5636 const int kHeight = 720;
5637 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005638 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005639 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5640 // Frame should not be dropped.
5641 WaitForEncodedFrame(1);
5642
5643 // Trigger QVGA "singlecast"
5644 // Update the config.
5645 VideoEncoderConfig video_encoder_config;
5646 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5647 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005648 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005649 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005650 "VP8", /*max qp*/ 56, /*screencast*/ false,
5651 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005652 for (auto& layer : video_encoder_config.simulcast_layers) {
5653 layer.num_temporal_layers = 1;
5654 layer.max_framerate = kDefaultFramerate;
5655 }
Asa Persson606d3cb2021-10-04 10:07:11 +02005656 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005657 video_encoder_config.content_type =
5658 VideoEncoderConfig::ContentType::kRealtimeVideo;
5659
5660 video_encoder_config.simulcast_layers[0].active = true;
5661 video_encoder_config.simulcast_layers[1].active = false;
5662 video_encoder_config.simulcast_layers[2].active = false;
5663
5664 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5665 kMaxPayloadLength);
5666 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5667
5668 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5669 // Frame should not be dropped.
5670 WaitForEncodedFrame(2);
5671
5672 // Trigger HD "singlecast"
5673 video_encoder_config.simulcast_layers[0].active = false;
5674 video_encoder_config.simulcast_layers[1].active = false;
5675 video_encoder_config.simulcast_layers[2].active = true;
5676
5677 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5678 kMaxPayloadLength);
5679 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5680
5681 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5682 // Frame should be dropped because of initial frame drop.
5683 ExpectDroppedFrame();
5684
5685 // Expect the sink_wants to specify a scaled frame.
5686 EXPECT_TRUE_WAIT(
5687 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5688 video_stream_encoder_->Stop();
5689}
5690
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005691TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005692 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005693 // Set simulcast.
5694 ResetEncoder("VP9", 1, 1, 3, false);
5695 fake_encoder_.SetQualityScaling(true);
5696 const int kWidth = 1280;
5697 const int kHeight = 720;
5698 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005699 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005700 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5701 // Frame should not be dropped.
5702 WaitForEncodedFrame(1);
5703
5704 // Trigger QVGA "singlecast"
5705 // Update the config.
5706 VideoEncoderConfig video_encoder_config;
5707 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5708 &video_encoder_config);
5709 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5710 vp9_settings.numberOfSpatialLayers = 3;
5711 // Since only one layer is active - automatic resize should be enabled.
5712 vp9_settings.automaticResizeOn = true;
5713 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005714 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005715 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005716 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005717 video_encoder_config.content_type =
5718 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 16:22:11 +02005719 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005720 // which SVC layers are active.
5721 video_encoder_config.simulcast_layers.resize(3);
5722
5723 video_encoder_config.simulcast_layers[0].active = true;
5724 video_encoder_config.simulcast_layers[1].active = false;
5725 video_encoder_config.simulcast_layers[2].active = false;
5726
5727 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5728 kMaxPayloadLength);
5729 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5730
5731 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5732 // Frame should not be dropped.
5733 WaitForEncodedFrame(2);
5734
5735 // Trigger HD "singlecast"
5736 video_encoder_config.simulcast_layers[0].active = false;
5737 video_encoder_config.simulcast_layers[1].active = false;
5738 video_encoder_config.simulcast_layers[2].active = true;
5739
5740 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5741 kMaxPayloadLength);
5742 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5743
5744 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5745 // Frame should be dropped because of initial frame drop.
5746 ExpectDroppedFrame();
5747
5748 // Expect the sink_wants to specify a scaled frame.
5749 EXPECT_TRUE_WAIT(
5750 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5751 video_stream_encoder_->Stop();
5752}
5753
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005754TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005755 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5756 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5757 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5758 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5759 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5760 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5761 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5762 fake_encoder_.SetResolutionBitrateLimits(
5763 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5764
5765 VideoEncoderConfig video_encoder_config;
5766 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5767 &video_encoder_config);
5768 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5769 vp9_settings.numberOfSpatialLayers = 3;
5770 // Since only one layer is active - automatic resize should be enabled.
5771 vp9_settings.automaticResizeOn = true;
5772 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005773 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005774 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005775 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005776 video_encoder_config.content_type =
5777 VideoEncoderConfig::ContentType::kRealtimeVideo;
5778 // Simulcast layers are used to indicate which spatial layers are active.
5779 video_encoder_config.simulcast_layers.resize(3);
5780 video_encoder_config.simulcast_layers[0].active = false;
5781 video_encoder_config.simulcast_layers[1].active = true;
5782 video_encoder_config.simulcast_layers[2].active = false;
5783
5784 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5785 kMaxPayloadLength);
5786 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5787
5788 // The encoder bitrate limits for 360p should be used.
5789 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5790 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005791 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5792 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5793 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5794 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5795 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5796 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005797 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005798 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005799 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005800 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005801
5802 // The encoder bitrate limits for 270p should be used.
5803 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5804 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005805 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5806 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5807 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5808 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5809 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5810 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005811 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005812 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005813 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005814 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005815
5816 video_stream_encoder_->Stop();
5817}
5818
5819TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005820 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5821 VideoEncoderConfig video_encoder_config;
5822 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5823 &video_encoder_config);
5824 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5825 vp9_settings.numberOfSpatialLayers = 3;
5826 // Since only one layer is active - automatic resize should be enabled.
5827 vp9_settings.automaticResizeOn = true;
5828 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005829 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005830 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005831 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005832 video_encoder_config.content_type =
5833 VideoEncoderConfig::ContentType::kRealtimeVideo;
5834 // Simulcast layers are used to indicate which spatial layers are active.
5835 video_encoder_config.simulcast_layers.resize(3);
5836 video_encoder_config.simulcast_layers[0].active = false;
5837 video_encoder_config.simulcast_layers[1].active = true;
5838 video_encoder_config.simulcast_layers[2].active = false;
5839
5840 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5841 kMaxPayloadLength);
5842 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5843
5844 // The default bitrate limits for 360p should be used.
5845 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005846 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5847 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005848 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5849 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005850 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5851 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5852 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5853 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5854 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5855 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005856 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005857 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005858 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005859 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005860
5861 // The default bitrate limits for 270p should be used.
5862 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005863 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5864 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005865 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5866 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005867 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5868 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5869 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5870 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5871 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5872 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005873 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005874 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005875 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005876 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005877
5878 video_stream_encoder_->Stop();
5879}
5880
5881TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005882 webrtc::test::ScopedKeyValueConfig field_trials(
5883 field_trials_, "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
Åsa Persson258e9892021-02-25 10:39:51 +01005884 VideoEncoderConfig video_encoder_config;
5885 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5886 &video_encoder_config);
5887 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5888 vp9_settings.numberOfSpatialLayers = 3;
5889 // Since only one layer is active - automatic resize should be enabled.
5890 vp9_settings.automaticResizeOn = true;
5891 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005892 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005893 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005894 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005895 video_encoder_config.content_type =
5896 VideoEncoderConfig::ContentType::kRealtimeVideo;
5897 // Simulcast layers are used to indicate which spatial layers are active.
5898 video_encoder_config.simulcast_layers.resize(3);
5899 video_encoder_config.simulcast_layers[0].active = false;
5900 video_encoder_config.simulcast_layers[1].active = true;
5901 video_encoder_config.simulcast_layers[2].active = false;
5902
5903 // Reset encoder for field trials to take effect.
5904 ConfigureEncoder(video_encoder_config.Copy());
5905
5906 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5907 kMaxPayloadLength);
5908 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5909
5910 // The default bitrate limits for 360p should not be used.
5911 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005912 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5913 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005914 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5915 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005916 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5917 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5918 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5919 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5920 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5921 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005922 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005923 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005924
5925 video_stream_encoder_->Stop();
5926}
5927
5928TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5929 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5930 /*num_spatial_layers=*/1, /*screenshare=*/false);
5931
5932 // The default singlecast bitrate limits for 720p should not be used.
5933 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005934 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5935 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005936 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5937 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005938 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5939 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5940 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5941 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5942 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5943 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005944 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005945 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005946
5947 video_stream_encoder_->Stop();
5948}
5949
5950TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005951 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5952 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5953 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5954 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5955 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5956 fake_encoder_.SetResolutionBitrateLimits(
5957 {kEncoderLimits180p, kEncoderLimits720p});
5958
5959 VideoEncoderConfig video_encoder_config;
5960 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5961 &video_encoder_config);
5962 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5963 vp9_settings.numberOfSpatialLayers = 3;
5964 // Since only one layer is active - automatic resize should be enabled.
5965 vp9_settings.automaticResizeOn = true;
5966 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005967 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005968 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005969 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005970 video_encoder_config.content_type =
5971 VideoEncoderConfig::ContentType::kRealtimeVideo;
5972 // Simulcast layers are used to indicate which spatial layers are active.
5973 video_encoder_config.simulcast_layers.resize(3);
5974 video_encoder_config.simulcast_layers[0].active = true;
5975 video_encoder_config.simulcast_layers[1].active = false;
5976 video_encoder_config.simulcast_layers[2].active = false;
5977
5978 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5979 kMaxPayloadLength);
5980 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5981
5982 // Limits not applied on lowest stream, limits for 180p should not be used.
5983 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5984 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005985 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5986 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5987 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5988 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5989 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5990 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005991 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005992 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005993 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005994 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005995
5996 video_stream_encoder_->Stop();
5997}
5998
5999TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006000 InitialFrameDropActivatesWhenResolutionIncreases) {
6001 const int kWidth = 640;
6002 const int kHeight = 360;
6003
6004 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006005 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006006 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
6007 // Frame should not be dropped.
6008 WaitForEncodedFrame(1);
6009
6010 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006011 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006012 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
6013 // Frame should not be dropped, bitrate not too low for frame.
6014 WaitForEncodedFrame(2);
6015
6016 // Incoming resolution increases.
6017 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
6018 // Expect to drop this frame, bitrate too low for frame.
6019 ExpectDroppedFrame();
6020
6021 // Expect the sink_wants to specify a scaled frame.
6022 EXPECT_TRUE_WAIT(
6023 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
6024 video_stream_encoder_->Stop();
6025}
6026
6027TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
6028 const int kWidth = 640;
6029 const int kHeight = 360;
6030 // So that quality scaling doesn't happen by itself.
6031 fake_encoder_.SetQp(kQpHigh);
6032
6033 AdaptingFrameForwarder source(&time_controller_);
6034 source.set_adaptation_enabled(true);
6035 video_stream_encoder_->SetSource(
6036 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
6037
6038 int timestamp = 1;
6039
6040 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006041 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006042 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6043 WaitForEncodedFrame(timestamp);
6044 timestamp += 9000;
6045 // Long pause to disable all first BWE drop logic.
6046 AdvanceTime(TimeDelta::Millis(1000));
6047
6048 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006049 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006050 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6051 // Not dropped frame, as initial frame drop is disabled by now.
6052 WaitForEncodedFrame(timestamp);
6053 timestamp += 9000;
6054 AdvanceTime(TimeDelta::Millis(100));
6055
6056 // Quality adaptation down.
6057 video_stream_encoder_->TriggerQualityLow();
6058
6059 // Adaptation has an effect.
6060 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6061 5000);
6062
6063 // Frame isn't dropped as initial frame dropper is disabled.
6064 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6065 WaitForEncodedFrame(timestamp);
6066 timestamp += 9000;
6067 AdvanceTime(TimeDelta::Millis(100));
6068
6069 // Quality adaptation up.
6070 video_stream_encoder_->TriggerQualityHigh();
6071
6072 // Adaptation has an effect.
6073 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
6074 5000);
6075
6076 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6077 // Frame should not be dropped, as initial framedropper is off.
6078 WaitForEncodedFrame(timestamp);
6079
6080 video_stream_encoder_->Stop();
6081}
6082
Åsa Persson7f354f82021-02-04 15:52:15 +01006083TEST_F(VideoStreamEncoderTest,
6084 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
6085 const int kMinStartBps360p = 222000;
6086 fake_encoder_.SetResolutionBitrateLimits(
6087 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6088 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6089 800000)});
6090
6091 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6092 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6093 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6094 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
6095 0, 0, 0);
6096 // Frame should not be dropped, bitrate not too low for frame.
6097 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6098 WaitForEncodedFrame(1);
6099
6100 // Incoming resolution increases, initial frame drop activates.
6101 // Frame should be dropped, link allocation too low for frame.
6102 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6103 ExpectDroppedFrame();
6104
6105 // Expect sink_wants to specify a scaled frame.
6106 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6107 5000);
6108 video_stream_encoder_->Stop();
6109}
6110
6111TEST_F(VideoStreamEncoderTest,
6112 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6113 const int kMinStartBps360p = 222000;
6114 fake_encoder_.SetResolutionBitrateLimits(
6115 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6116 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6117 800000)});
6118
6119 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6120 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6121 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6122 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6123 0, 0, 0);
6124 // Frame should not be dropped, bitrate not too low for frame.
6125 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6126 WaitForEncodedFrame(1);
6127
6128 // Incoming resolution increases, initial frame drop activates.
6129 // Frame should be dropped, link allocation not too low for frame.
6130 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6131 WaitForEncodedFrame(2);
6132
6133 video_stream_encoder_->Stop();
6134}
6135
Åsa Perssone644a032019-11-08 15:56:00 +01006136TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006137 webrtc::test::ScopedKeyValueConfig field_trials(
6138 field_trials_,
Åsa Persson06defc42021-09-10 15:28:48 +02006139 "WebRTC-Video-QualityRampupSettings/"
6140 "min_pixels:921600,min_duration_ms:2000/");
6141
6142 const int kWidth = 1280;
6143 const int kHeight = 720;
6144 const int kFps = 10;
6145 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006146
6147 // Reset encoder for field trials to take effect.
6148 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 10:07:11 +02006149 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 11:34:32 +02006150 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01006151 ConfigureEncoder(std::move(config));
6152 fake_encoder_.SetQp(kQpLow);
6153
6154 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006155 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01006156 source.set_adaptation_enabled(true);
6157 video_stream_encoder_->SetSource(&source,
6158 DegradationPreference::MAINTAIN_FRAMERATE);
6159
6160 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 10:07:11 +02006161 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 18:49:07 +02006162 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006163 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006164
6165 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 15:28:48 +02006166 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006167 int64_t timestamp_ms = kFrameIntervalMs;
6168 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6169 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02006170 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6171 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01006172
6173 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02006174 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6175 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006176
Artem Titovab30d722021-07-27 16:22:11 +02006177 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 15:28:48 +02006178 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 15:56:00 +01006179 for (size_t i = 1; i <= 10; i++) {
6180 timestamp_ms += kFrameIntervalMs;
6181 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6182 WaitForEncodedFrame(timestamp_ms);
6183 }
Åsa Persson06defc42021-09-10 15:28:48 +02006184
6185 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6186 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6187 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6188 timestamp_ms += kFrameIntervalMs;
6189 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6190 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 15:56:00 +01006191 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6192 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6193
Åsa Persson06defc42021-09-10 15:28:48 +02006194 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 15:56:00 +01006195 timestamp_ms += kFrameIntervalMs;
6196 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6197 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006198 // The ramp-up code involves the adaptation queue, give it time to execute.
6199 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006200 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006201 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006202
6203 // Frame should not be adapted.
6204 timestamp_ms += kFrameIntervalMs;
6205 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6206 WaitForEncodedFrame(kWidth, kHeight);
6207 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6208
6209 video_stream_encoder_->Stop();
6210}
6211
mflodmancc3d4422017-08-03 08:27:51 -07006212TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006213 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006214 webrtc::test::ScopedKeyValueConfig field_trials(
6215 field_trials_, "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006216 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006217 source.set_adaptation_enabled(true);
6218 video_stream_encoder_->SetSource(&source,
6219 DegradationPreference::MAINTAIN_FRAMERATE);
6220 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006221 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006222 fake_encoder_.SetQp(kQpHigh + 1);
6223 const int kWidth = 1280;
6224 const int kHeight = 720;
6225 const int64_t kFrameIntervalMs = 100;
6226 int64_t timestamp_ms = kFrameIntervalMs;
6227 for (size_t i = 1; i <= 100; i++) {
6228 timestamp_ms += kFrameIntervalMs;
6229 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6230 WaitForEncodedFrame(timestamp_ms);
6231 }
6232 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6233 // for the first time.
6234 // TODO(eshr): We should avoid these waits by using threads with simulated
6235 // time.
6236 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6237 2000 * 2.5 * 2);
6238 timestamp_ms += kFrameIntervalMs;
6239 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6240 WaitForEncodedFrame(timestamp_ms);
6241 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6242 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6243 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6244
6245 // Disable Quality scaling by turning off scaler on the encoder and
6246 // reconfiguring.
6247 fake_encoder_.SetQualityScaling(false);
6248 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6249 kMaxPayloadLength);
6250 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Markus Handell28c71802021-11-08 10:11:55 +01006251 AdvanceTime(TimeDelta::Zero());
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006252 // Since we turned off the quality scaler, the adaptations made by it are
6253 // removed.
6254 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6255 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6256
6257 video_stream_encoder_->Stop();
6258}
6259
6260TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006261 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6262 const int kTooSmallWidth = 10;
6263 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006264 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006265 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006266
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006267 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006268 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006269 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006270 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006271 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006272 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6273
6274 // Trigger adapt down, too small frame, expect no change.
6275 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006276 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006277 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006278 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006279 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6280 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6281
mflodmancc3d4422017-08-03 08:27:51 -07006282 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006283}
6284
mflodmancc3d4422017-08-03 08:27:51 -07006285TEST_F(VideoStreamEncoderTest,
6286 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006287 const int kTooSmallWidth = 10;
6288 const int kTooSmallHeight = 10;
6289 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006290 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006291 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006292
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006293 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006294 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006295 video_stream_encoder_->SetSource(&source,
6296 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006297 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006298 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6299 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6300
6301 // Trigger adapt down, expect limited framerate.
6302 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006303 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006304 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006305 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006306 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6307 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6308 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6309
6310 // Trigger adapt down, too small frame, expect no change.
6311 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006312 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006313 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006314 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006315 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6316 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6317 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6318
mflodmancc3d4422017-08-03 08:27:51 -07006319 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006320}
6321
mflodmancc3d4422017-08-03 08:27:51 -07006322TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006323 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006324 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006325 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006326 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006327 const int kFrameWidth = 1280;
6328 const int kFrameHeight = 720;
6329 video_source_.IncomingCapturedFrame(
6330 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006331 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006332 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006333}
6334
sprangb1ca0732017-02-01 08:38:12 -08006335// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006336TEST_F(VideoStreamEncoderTest,
6337 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006338 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006339 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006340
6341 const int kFrameWidth = 1280;
6342 const int kFrameHeight = 720;
6343 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006344 // requested by
6345 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006346 video_source_.set_adaptation_enabled(true);
6347
6348 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006349 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006350 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006351
6352 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006353 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006354 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006355 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006356 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006357
asaperssonfab67072017-04-04 05:51:49 -07006358 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006359 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006360 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006361 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006362 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006363
mflodmancc3d4422017-08-03 08:27:51 -07006364 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006365}
sprangfe627f32017-03-29 08:24:59 -07006366
mflodmancc3d4422017-08-03 08:27:51 -07006367TEST_F(VideoStreamEncoderTest,
6368 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006369 const int kFrameWidth = 1280;
6370 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006371
Henrik Boström381d1092020-05-12 18:49:07 +02006372 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006373 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006374 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006375 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006376 video_source_.set_adaptation_enabled(true);
6377
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006378 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006379
6380 video_source_.IncomingCapturedFrame(
6381 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006382 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006383
6384 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006385 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006386
6387 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006388 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006389 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006390 video_source_.IncomingCapturedFrame(
6391 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006392 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006393 }
6394
6395 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006396 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006397 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006398 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006399 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006400 video_source_.IncomingCapturedFrame(
6401 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006402 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006403 ++num_frames_dropped;
6404 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006405 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006406 }
6407 }
6408
sprang4847ae62017-06-27 07:06:52 -07006409 // Add some slack to account for frames dropped by the frame dropper.
6410 const int kErrorMargin = 1;
6411 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006412 kErrorMargin);
6413
6414 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006415 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006416 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006417 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006418 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006419 video_source_.IncomingCapturedFrame(
6420 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006421 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006422 ++num_frames_dropped;
6423 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006424 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006425 }
6426 }
sprang4847ae62017-06-27 07:06:52 -07006427 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006428 kErrorMargin);
6429
6430 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006431 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006432 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006433 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006434 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006435 video_source_.IncomingCapturedFrame(
6436 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006437 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006438 ++num_frames_dropped;
6439 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006440 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006441 }
6442 }
sprang4847ae62017-06-27 07:06:52 -07006443 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006444 kErrorMargin);
6445
6446 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006447 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006448 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006449 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006450 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006451 video_source_.IncomingCapturedFrame(
6452 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006453 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006454 ++num_frames_dropped;
6455 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006456 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006457 }
6458 }
6459 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6460
mflodmancc3d4422017-08-03 08:27:51 -07006461 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006462}
6463
mflodmancc3d4422017-08-03 08:27:51 -07006464TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006465 const int kFramerateFps = 5;
6466 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006467 const int kFrameWidth = 1280;
6468 const int kFrameHeight = 720;
6469
sprang4847ae62017-06-27 07:06:52 -07006470 // Reconfigure encoder with two temporal layers and screensharing, which will
6471 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006472 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006473
Henrik Boström381d1092020-05-12 18:49:07 +02006474 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006475 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006476 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006477 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006478 video_source_.set_adaptation_enabled(true);
6479
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006480 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006481
6482 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006483 rtc::VideoSinkWants last_wants;
6484 do {
6485 last_wants = video_source_.sink_wants();
6486
sprangc5d62e22017-04-02 23:53:04 -07006487 // Insert frames to get a new fps estimate...
6488 for (int j = 0; j < kFramerateFps; ++j) {
6489 video_source_.IncomingCapturedFrame(
6490 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006491 if (video_source_.last_sent_width()) {
6492 sink_.WaitForEncodedFrame(timestamp_ms);
6493 }
sprangc5d62e22017-04-02 23:53:04 -07006494 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006495 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006496 }
6497 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006498 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006499 } while (video_source_.sink_wants().max_framerate_fps <
6500 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006501
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006502 EXPECT_THAT(video_source_.sink_wants(),
6503 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006504
mflodmancc3d4422017-08-03 08:27:51 -07006505 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006506}
asaperssonf7e294d2017-06-13 23:25:22 -07006507
mflodmancc3d4422017-08-03 08:27:51 -07006508TEST_F(VideoStreamEncoderTest,
6509 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006510 const int kWidth = 1280;
6511 const int kHeight = 720;
6512 const int64_t kFrameIntervalMs = 150;
6513 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006514 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006515 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006516
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006517 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006518 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006519 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006520 video_stream_encoder_->SetSource(&source,
6521 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006522 timestamp_ms += kFrameIntervalMs;
6523 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006524 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006525 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006526 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6527 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6528 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6529
6530 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006531 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006532 timestamp_ms += kFrameIntervalMs;
6533 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006534 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006535 EXPECT_THAT(source.sink_wants(),
6536 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006537 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6538 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6539 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6540
6541 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006542 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006543 timestamp_ms += kFrameIntervalMs;
6544 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006545 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006546 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006547 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6548 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6549 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6550
6551 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006552 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006553 timestamp_ms += kFrameIntervalMs;
6554 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006555 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006556 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006557 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6558 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6559 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6560
6561 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006562 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006563 timestamp_ms += kFrameIntervalMs;
6564 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006565 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006566 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006567 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6568 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6569 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6570
6571 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006572 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006573 timestamp_ms += kFrameIntervalMs;
6574 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006575 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006576 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006577 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6578 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6579 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6580
6581 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006582 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006583 timestamp_ms += kFrameIntervalMs;
6584 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006585 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006586 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006587 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6588 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6589 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6590
6591 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006592 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006593 timestamp_ms += kFrameIntervalMs;
6594 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006595 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006596 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006597 rtc::VideoSinkWants last_wants = source.sink_wants();
6598 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6599 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6600 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6601
6602 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006603 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006604 timestamp_ms += kFrameIntervalMs;
6605 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006606 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006607 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006608 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6609 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6610 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6611
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006612 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006613 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006614 timestamp_ms += kFrameIntervalMs;
6615 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006616 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006617 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006618 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6619 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6620 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6621
6622 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006623 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006624 timestamp_ms += kFrameIntervalMs;
6625 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006626 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006627 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006628 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6629 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6630 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6631
6632 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006633 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006634 timestamp_ms += kFrameIntervalMs;
6635 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006636 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006637 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006638 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6639 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6640 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6641
6642 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006643 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006644 timestamp_ms += kFrameIntervalMs;
6645 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006646 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006647 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006648 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6649 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6650 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6651
6652 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006653 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006654 timestamp_ms += kFrameIntervalMs;
6655 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006656 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006657 EXPECT_THAT(source.sink_wants(), FpsMax());
6658 EXPECT_EQ(source.sink_wants().max_pixel_count,
6659 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006660 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6661 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6662 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6663
6664 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006665 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006666 timestamp_ms += kFrameIntervalMs;
6667 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006668 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006669 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006670 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6671 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6672 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6673
Åsa Persson30ab0152019-08-27 12:22:33 +02006674 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006675 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006676 timestamp_ms += kFrameIntervalMs;
6677 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006678 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006679 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006680 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006681 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6682 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6683 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6684
6685 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006686 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006687 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006688 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6689
mflodmancc3d4422017-08-03 08:27:51 -07006690 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006691}
6692
mflodmancc3d4422017-08-03 08:27:51 -07006693TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006694 const int kWidth = 1280;
6695 const int kHeight = 720;
6696 const int64_t kFrameIntervalMs = 150;
6697 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006698 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006699 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006700
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006701 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006702 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006703 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006704 video_stream_encoder_->SetSource(&source,
6705 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006706 timestamp_ms += kFrameIntervalMs;
6707 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006708 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006709 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006710 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6711 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6712 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6713 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6714 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6715 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6716
6717 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006718 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006719 timestamp_ms += kFrameIntervalMs;
6720 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006721 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006722 EXPECT_THAT(source.sink_wants(),
6723 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006724 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6725 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6726 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6727 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6728 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6729 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6730
6731 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006732 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006733 timestamp_ms += kFrameIntervalMs;
6734 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006735 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006736 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006737 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6738 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6739 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6740 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6741 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6742 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6743
6744 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006745 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006746 timestamp_ms += kFrameIntervalMs;
6747 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006748 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006749 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006750 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006751 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6752 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6753 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6754 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6755 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6756
Evan Shrubsole64469032020-06-11 10:45:29 +02006757 // Trigger cpu adapt up, expect no change since QP is most limited.
6758 {
6759 // Store current sink wants since we expect no change and if there is no
6760 // change then last_wants() is not updated.
6761 auto previous_sink_wants = source.sink_wants();
6762 video_stream_encoder_->TriggerCpuUnderuse();
6763 timestamp_ms += kFrameIntervalMs;
6764 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6765 WaitForEncodedFrame(timestamp_ms);
6766 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6767 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6768 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6769 }
6770
6771 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6772 video_stream_encoder_->TriggerQualityHigh();
6773 timestamp_ms += kFrameIntervalMs;
6774 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6775 WaitForEncodedFrame(timestamp_ms);
6776 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6777 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6778 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6779 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6780 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6781 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6782 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6783
6784 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6785 // expect increased resolution (960x540@30fps).
6786 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006787 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006788 timestamp_ms += kFrameIntervalMs;
6789 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006790 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006791 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006792 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6793 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6794 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6795 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6796 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006797 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006798
Evan Shrubsole64469032020-06-11 10:45:29 +02006799 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6800 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006801 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006802 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006803 timestamp_ms += kFrameIntervalMs;
6804 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006805 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006806 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006807 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006808 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6809 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6810 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6811 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6812 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006813 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006814
6815 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006816 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006817 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006818 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006819 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006820
mflodmancc3d4422017-08-03 08:27:51 -07006821 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006822}
6823
mflodmancc3d4422017-08-03 08:27:51 -07006824TEST_F(VideoStreamEncoderTest,
6825 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006826 const int kWidth = 640;
6827 const int kHeight = 360;
6828 const int kFpsLimit = 15;
6829 const int64_t kFrameIntervalMs = 150;
6830 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006831 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006832 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006833
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006834 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006835 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006836 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006837 video_stream_encoder_->SetSource(&source,
6838 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006839 timestamp_ms += kFrameIntervalMs;
6840 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006841 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006842 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006843 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6844 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6845 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6846 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6847 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6848 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6849
6850 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006851 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006852 timestamp_ms += kFrameIntervalMs;
6853 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006854 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006855 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006856 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6857 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6858 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6859 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6860 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6861 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6862
6863 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006864 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006865 timestamp_ms += kFrameIntervalMs;
6866 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006867 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006868 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006869 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006870 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006871 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6872 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6873 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6874 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6875
Evan Shrubsole64469032020-06-11 10:45:29 +02006876 // Trigger cpu adapt up, expect no change because quality is most limited.
6877 {
6878 auto previous_sink_wants = source.sink_wants();
6879 // Store current sink wants since we expect no change ind if there is no
6880 // change then last__wants() is not updated.
6881 video_stream_encoder_->TriggerCpuUnderuse();
6882 timestamp_ms += kFrameIntervalMs;
6883 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6884 WaitForEncodedFrame(timestamp_ms);
6885 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6886 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6887 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6888 }
6889
6890 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6891 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006892 timestamp_ms += kFrameIntervalMs;
6893 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006894 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006895 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006896 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6897 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6898 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006899 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6900 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6901 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006902
Evan Shrubsole64469032020-06-11 10:45:29 +02006903 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006904 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006905 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006906 timestamp_ms += kFrameIntervalMs;
6907 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006908 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006909 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006910 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6911 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6912 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6913 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6914 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006915 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006916
6917 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006918 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006919 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006920 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006921 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006922
mflodmancc3d4422017-08-03 08:27:51 -07006923 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006924}
6925
mflodmancc3d4422017-08-03 08:27:51 -07006926TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006927 const int kFrameWidth = 1920;
6928 const int kFrameHeight = 1080;
Ying Wangdb0d5862022-04-28 19:26:22 +02006929 // 2/3 of 1920.
6930 const int kAdaptedFrameWidth = 1280;
6931 // 2/3 of 1080.
6932 const int kAdaptedFrameHeight = 720;
ilnik6b826ef2017-06-16 06:53:48 -07006933 const int kFramerate = 24;
6934
Henrik Boström381d1092020-05-12 18:49:07 +02006935 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006936 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006937 // Trigger reconfigure encoder (without resetting the entire instance).
6938 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006939 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6940 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02006941 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 06:53:48 -07006942 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006943 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006944 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006945 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006946 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006947
6948 video_source_.set_adaptation_enabled(true);
6949
6950 video_source_.IncomingCapturedFrame(
6951 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006952 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006953
6954 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006955 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006956 video_source_.IncomingCapturedFrame(
6957 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006958 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006959
mflodmancc3d4422017-08-03 08:27:51 -07006960 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006961}
6962
mflodmancc3d4422017-08-03 08:27:51 -07006963TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006964 const int kFrameWidth = 1280;
6965 const int kFrameHeight = 720;
6966 const int kLowFps = 2;
6967 const int kHighFps = 30;
6968
Henrik Boström381d1092020-05-12 18:49:07 +02006969 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006970 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006971
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006972 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006973 max_framerate_ = kLowFps;
6974
6975 // Insert 2 seconds of 2fps video.
6976 for (int i = 0; i < kLowFps * 2; ++i) {
6977 video_source_.IncomingCapturedFrame(
6978 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6979 WaitForEncodedFrame(timestamp_ms);
6980 timestamp_ms += 1000 / kLowFps;
6981 }
6982
6983 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006984 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006985 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006986 video_source_.IncomingCapturedFrame(
6987 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6988 WaitForEncodedFrame(timestamp_ms);
6989 timestamp_ms += 1000 / kLowFps;
6990
6991 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6992
6993 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006994 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006995 const int kFrameIntervalMs = 1000 / kHighFps;
6996 max_framerate_ = kHighFps;
6997 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6998 video_source_.IncomingCapturedFrame(
6999 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7000 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
7001 // be dropped if the encoder hans't been updated with the new higher target
7002 // framerate yet, causing it to overshoot the target bitrate and then
7003 // suffering the wrath of the media optimizer.
7004 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
7005 timestamp_ms += kFrameIntervalMs;
7006 }
7007
7008 // Don expect correct measurement just yet, but it should be higher than
7009 // before.
7010 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
7011
mflodmancc3d4422017-08-03 08:27:51 -07007012 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007013}
7014
mflodmancc3d4422017-08-03 08:27:51 -07007015TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07007016 const int kFrameWidth = 1280;
7017 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 15:09:05 +02007018 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01007019 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02007020 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07007021
Henrik Boström381d1092020-05-12 18:49:07 +02007022 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007023 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07007024 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07007025
7026 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007027 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07007028 video_source_.IncomingCapturedFrame(
7029 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7030 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02007031 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007032
7033 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02007034 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007035 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07007036
7037 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02007038 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007039 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07007040
Per Kjellanderdcef6412020-10-07 15:09:05 +02007041 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07007042 video_source_.IncomingCapturedFrame(
7043 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7044 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02007045 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007046
mflodmancc3d4422017-08-03 08:27:51 -07007047 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007048}
ilnik6b826ef2017-06-16 06:53:48 -07007049
Niels Möller4db138e2018-04-19 09:04:13 +02007050TEST_F(VideoStreamEncoderTest,
7051 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
7052 const int kFrameWidth = 1280;
7053 const int kFrameHeight = 720;
Markus Handell8e4197b2022-05-30 15:45:28 +02007054 const test::ScopedKeyValueConfig kFieldTrials;
7055 const CpuOveruseOptions default_options(kFieldTrials);
Henrik Boström381d1092020-05-12 18:49:07 +02007056 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007057 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007058 video_source_.IncomingCapturedFrame(
7059 CreateFrame(1, kFrameWidth, kFrameHeight));
7060 WaitForEncodedFrame(1);
7061 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7062 .low_encode_usage_threshold_percent,
7063 default_options.low_encode_usage_threshold_percent);
7064 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7065 .high_encode_usage_threshold_percent,
7066 default_options.high_encode_usage_threshold_percent);
7067 video_stream_encoder_->Stop();
7068}
7069
7070TEST_F(VideoStreamEncoderTest,
7071 HigherCpuAdaptationThresholdsForHardwareEncoder) {
7072 const int kFrameWidth = 1280;
7073 const int kFrameHeight = 720;
Markus Handell8e4197b2022-05-30 15:45:28 +02007074 const test::ScopedKeyValueConfig kFieldTrials;
7075 CpuOveruseOptions hardware_options(kFieldTrials);
Niels Möller4db138e2018-04-19 09:04:13 +02007076 hardware_options.low_encode_usage_threshold_percent = 150;
7077 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01007078 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02007079
Henrik Boström381d1092020-05-12 18:49:07 +02007080 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007081 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007082 video_source_.IncomingCapturedFrame(
7083 CreateFrame(1, kFrameWidth, kFrameHeight));
7084 WaitForEncodedFrame(1);
7085 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7086 .low_encode_usage_threshold_percent,
7087 hardware_options.low_encode_usage_threshold_percent);
7088 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7089 .high_encode_usage_threshold_percent,
7090 hardware_options.high_encode_usage_threshold_percent);
7091 video_stream_encoder_->Stop();
7092}
7093
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007094TEST_F(VideoStreamEncoderTest,
7095 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
7096 const int kFrameWidth = 1280;
7097 const int kFrameHeight = 720;
7098
Markus Handell8e4197b2022-05-30 15:45:28 +02007099 const test::ScopedKeyValueConfig kFieldTrials;
7100 const CpuOveruseOptions default_options(kFieldTrials);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007101 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007102 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007103 video_source_.IncomingCapturedFrame(
7104 CreateFrame(1, kFrameWidth, kFrameHeight));
7105 WaitForEncodedFrame(1);
7106 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7107 .low_encode_usage_threshold_percent,
7108 default_options.low_encode_usage_threshold_percent);
7109 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7110 .high_encode_usage_threshold_percent,
7111 default_options.high_encode_usage_threshold_percent);
7112
Markus Handell8e4197b2022-05-30 15:45:28 +02007113 CpuOveruseOptions hardware_options(kFieldTrials);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007114 hardware_options.low_encode_usage_threshold_percent = 150;
7115 hardware_options.high_encode_usage_threshold_percent = 200;
7116 fake_encoder_.SetIsHardwareAccelerated(true);
7117
7118 video_source_.IncomingCapturedFrame(
7119 CreateFrame(2, kFrameWidth, kFrameHeight));
7120 WaitForEncodedFrame(2);
7121
7122 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7123 .low_encode_usage_threshold_percent,
7124 hardware_options.low_encode_usage_threshold_percent);
7125 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7126 .high_encode_usage_threshold_percent,
7127 hardware_options.high_encode_usage_threshold_percent);
7128
7129 video_stream_encoder_->Stop();
7130}
7131
Niels Möller6bb5ab92019-01-11 11:11:10 +01007132TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7133 const int kFrameWidth = 320;
7134 const int kFrameHeight = 240;
7135 const int kFps = 30;
Asa Persson606d3cb2021-10-04 10:07:11 +02007136 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007137 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7138
Henrik Boström381d1092020-05-12 18:49:07 +02007139 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007140 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007141
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007142 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007143 max_framerate_ = kFps;
7144
7145 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7146 fake_encoder_.SimulateOvershoot(1.0);
7147 int num_dropped = 0;
7148 for (int i = 0; i < kNumFramesInRun; ++i) {
7149 video_source_.IncomingCapturedFrame(
7150 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7151 // Wait up to two frame durations for a frame to arrive.
7152 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7153 ++num_dropped;
7154 }
7155 timestamp_ms += 1000 / kFps;
7156 }
7157
Erik Språnga8d48ab2019-02-08 14:17:40 +01007158 // Framerate should be measured to be near the expected target rate.
7159 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7160
7161 // Frame drops should be within 5% of expected 0%.
7162 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007163
7164 // Make encoder produce frames at double the expected bitrate during 3 seconds
7165 // of video, verify number of drops. Rate needs to be slightly changed in
7166 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01007167 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02007168 const RateControlSettings trials =
7169 RateControlSettings::ParseFromFieldTrials();
7170 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01007171 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02007172 // frame dropping since the adjuter will try to just lower the target
7173 // bitrate rather than drop frames. If network headroom can be used, it
7174 // doesn't push back as hard so we don't need quite as much overshoot.
7175 // These numbers are unfortunately a bit magical but there's not trivial
7176 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01007177 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01007178 }
7179 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02007180 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007181 kTargetBitrate + DataRate::KilobitsPerSec(1),
7182 kTargetBitrate + DataRate::KilobitsPerSec(1),
7183 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007184 num_dropped = 0;
7185 for (int i = 0; i < kNumFramesInRun; ++i) {
7186 video_source_.IncomingCapturedFrame(
7187 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7188 // Wait up to two frame durations for a frame to arrive.
7189 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7190 ++num_dropped;
7191 }
7192 timestamp_ms += 1000 / kFps;
7193 }
7194
Henrik Boström381d1092020-05-12 18:49:07 +02007195 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007196 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007197
7198 // Target framerate should be still be near the expected target, despite
7199 // the frame drops.
7200 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7201
7202 // Frame drops should be within 5% of expected 50%.
7203 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007204
7205 video_stream_encoder_->Stop();
7206}
7207
7208TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7209 const int kFrameWidth = 320;
7210 const int kFrameHeight = 240;
7211 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 10:07:11 +02007212 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007213
7214 ASSERT_GT(max_framerate_, kActualInputFps);
7215
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007216 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007217 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007218 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007219 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007220
7221 // Insert 3 seconds of video, with an input fps lower than configured max.
7222 for (int i = 0; i < kActualInputFps * 3; ++i) {
7223 video_source_.IncomingCapturedFrame(
7224 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7225 // Wait up to two frame durations for a frame to arrive.
7226 WaitForEncodedFrame(timestamp_ms);
7227 timestamp_ms += 1000 / kActualInputFps;
7228 }
7229
7230 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7231
7232 video_stream_encoder_->Stop();
7233}
7234
Markus Handell9a478b52021-11-18 16:07:01 +01007235TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007236 VideoFrame::UpdateRect rect;
Markus Handell9a478b52021-11-18 16:07:01 +01007237 test::FrameForwarder source;
7238 video_stream_encoder_->SetSource(&source,
7239 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02007240 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007241 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007242
Markus Handell9a478b52021-11-18 16:07:01 +01007243 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(1, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007244 WaitForEncodedFrame(1);
7245 // On the very first frame full update should be forced.
7246 rect = fake_encoder_.GetLastUpdateRect();
7247 EXPECT_EQ(rect.offset_x, 0);
7248 EXPECT_EQ(rect.offset_y, 0);
7249 EXPECT_EQ(rect.height, codec_height_);
7250 EXPECT_EQ(rect.width, codec_width_);
Markus Handell9a478b52021-11-18 16:07:01 +01007251 // Frame with NTP timestamp 2 will be dropped due to outstanding frames
7252 // scheduled for processing during encoder queue processing of frame 2.
7253 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(2, nullptr, 1));
7254 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(3, nullptr, 10));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007255 WaitForEncodedFrame(3);
7256 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7257 rect = fake_encoder_.GetLastUpdateRect();
7258 EXPECT_EQ(rect.offset_x, 1);
7259 EXPECT_EQ(rect.offset_y, 0);
7260 EXPECT_EQ(rect.width, 10);
7261 EXPECT_EQ(rect.height, 1);
7262
Markus Handell9a478b52021-11-18 16:07:01 +01007263 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(4, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007264 WaitForEncodedFrame(4);
7265 // Previous frame was encoded, so no accumulation should happen.
7266 rect = fake_encoder_.GetLastUpdateRect();
7267 EXPECT_EQ(rect.offset_x, 0);
7268 EXPECT_EQ(rect.offset_y, 0);
7269 EXPECT_EQ(rect.width, 1);
7270 EXPECT_EQ(rect.height, 1);
7271
7272 video_stream_encoder_->Stop();
7273}
7274
Erik Språngd7329ca2019-02-21 21:19:53 +01007275TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007276 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007277 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007278
7279 // First frame is always keyframe.
7280 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7281 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007282 EXPECT_THAT(
7283 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007284 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007285
7286 // Insert delta frame.
7287 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7288 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007289 EXPECT_THAT(
7290 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007291 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007292
7293 // Request next frame be a key-frame.
7294 video_stream_encoder_->SendKeyFrame();
7295 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7296 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007297 EXPECT_THAT(
7298 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007299 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007300
7301 video_stream_encoder_->Stop();
7302}
7303
7304TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7305 // Setup simulcast with three streams.
7306 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007307 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007308 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7309 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007310 // Wait for all three layers before triggering event.
7311 sink_.SetNumExpectedLayers(3);
7312
7313 // First frame is always keyframe.
7314 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7315 WaitForEncodedFrame(1);
7316 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007317 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7318 VideoFrameType::kVideoFrameKey,
7319 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007320
7321 // Insert delta frame.
7322 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7323 WaitForEncodedFrame(2);
7324 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007325 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7326 VideoFrameType::kVideoFrameDelta,
7327 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007328
7329 // Request next frame be a key-frame.
7330 // Only first stream is configured to produce key-frame.
7331 video_stream_encoder_->SendKeyFrame();
7332 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7333 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007334
7335 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7336 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007337 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007338 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007339 VideoFrameType::kVideoFrameKey,
7340 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007341
7342 video_stream_encoder_->Stop();
7343}
7344
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007345TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007346 // SPS contains VUI with restrictions on the maximum number of reordered
7347 // pictures, there is no need to rewrite the bitstream to enable faster
7348 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007349 ResetEncoder("H264", 1, 1, 1, false);
7350
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007351 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007352 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007353 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007354
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007355 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 10:07:11 +02007356 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007357
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007358 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7359 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007360
7361 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007362 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007363
7364 video_stream_encoder_->Stop();
7365}
7366
7367TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007368 // SPS does not contain VUI, the bitstream is will be rewritten with added
7369 // VUI with restrictions on the maximum number of reordered pictures to
7370 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007371 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7372 0x00, 0x00, 0x03, 0x03, 0xF4,
7373 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007374 ResetEncoder("H264", 1, 1, 1, false);
7375
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007376 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007377 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007378 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007379
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007380 fake_encoder_.SetEncodedImageData(
7381 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007382
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007383 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7384 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007385
7386 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007387 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007388
7389 video_stream_encoder_->Stop();
7390}
7391
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007392TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7393 const int kFrameWidth = 1280;
7394 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 10:07:11 +02007395 const DataRate kTargetBitrate =
7396 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007397
Henrik Boström381d1092020-05-12 18:49:07 +02007398 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007399 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007400 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7401
7402 // Insert a first video frame. It should be dropped because of downscale in
7403 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007404 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007405 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7406 frame.set_rotation(kVideoRotation_270);
7407 video_source_.IncomingCapturedFrame(frame);
7408
7409 ExpectDroppedFrame();
7410
7411 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007412 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007413 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7414 frame.set_rotation(kVideoRotation_90);
7415 video_source_.IncomingCapturedFrame(frame);
7416
7417 WaitForEncodedFrame(timestamp_ms);
7418 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7419
7420 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007421 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007422 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7423 frame.set_rotation(kVideoRotation_180);
7424 video_source_.IncomingCapturedFrame(frame);
7425
7426 WaitForEncodedFrame(timestamp_ms);
7427 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7428
7429 video_stream_encoder_->Stop();
7430}
7431
Erik Språng5056af02019-09-02 15:53:11 +02007432TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7433 const int kFrameWidth = 320;
7434 const int kFrameHeight = 180;
7435
7436 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007437 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007438 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7439 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7440 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007441 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007442 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007443 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007444
7445 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007446 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007447 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7448 frame.set_rotation(kVideoRotation_270);
7449 video_source_.IncomingCapturedFrame(frame);
7450 WaitForEncodedFrame(timestamp_ms);
7451
7452 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 10:07:11 +02007453 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007454 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7455 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007456 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007457 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007458 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007459 /*link_allocation=*/target_rate,
7460 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007461 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007462 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007463 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7464
7465 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7466 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7467 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007468 DataRate allocation_sum =
7469 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007470 EXPECT_EQ(min_rate, allocation_sum);
7471 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7472
7473 video_stream_encoder_->Stop();
7474}
7475
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007476TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007477 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007478 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007479 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007480 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007481 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7482 WaitForEncodedFrame(1);
7483
7484 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7485 ASSERT_TRUE(prev_rate_settings.has_value());
7486 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7487 kDefaultFramerate);
7488
7489 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7490 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7491 timestamp_ms += 1000 / kDefaultFramerate;
7492 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7493 WaitForEncodedFrame(timestamp_ms);
7494 }
7495 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7496 kDefaultFramerate);
7497 // Capture larger frame to trigger a reconfigure.
7498 codec_height_ *= 2;
7499 codec_width_ *= 2;
7500 timestamp_ms += 1000 / kDefaultFramerate;
7501 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7502 WaitForEncodedFrame(timestamp_ms);
7503
7504 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7505 auto current_rate_settings =
7506 fake_encoder_.GetAndResetLastRateControlSettings();
7507 // Ensure we have actually reconfigured twice
7508 // The rate settings should have been set again even though
7509 // they haven't changed.
7510 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007511 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007512
7513 video_stream_encoder_->Stop();
7514}
7515
philipeld9cc8c02019-09-16 14:53:40 +02007516struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007517 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007518 MOCK_METHOD(void,
7519 RequestEncoderSwitch,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007520 (const webrtc::SdpVideoFormat& format,
7521 bool allow_default_fallback),
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007522 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007523};
7524
philipel9b058032020-02-10 11:30:00 +01007525TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7526 constexpr int kDontCare = 100;
7527 StrictMock<MockEncoderSelector> encoder_selector;
7528 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7529 &fake_encoder_, &encoder_selector);
7530 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7531
7532 // Reset encoder for new configuration to take effect.
7533 ConfigureEncoder(video_encoder_config_.Copy());
7534
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007535 EXPECT_CALL(encoder_selector, OnCurrentEncoder);
philipel9b058032020-02-10 11:30:00 +01007536
7537 video_source_.IncomingCapturedFrame(
7538 CreateFrame(kDontCare, kDontCare, kDontCare));
Markus Handell28c71802021-11-08 10:11:55 +01007539 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007540 video_stream_encoder_->Stop();
7541
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007542 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007543 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007544 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7545 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007546 video_stream_encoder_.reset();
7547}
7548
7549TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7550 constexpr int kDontCare = 100;
7551
7552 NiceMock<MockEncoderSelector> encoder_selector;
7553 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7554 video_send_config_.encoder_settings.encoder_switch_request_callback =
7555 &switch_callback;
7556 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7557 &fake_encoder_, &encoder_selector);
7558 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7559
7560 // Reset encoder for new configuration to take effect.
7561 ConfigureEncoder(video_encoder_config_.Copy());
7562
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007563 ON_CALL(encoder_selector, OnAvailableBitrate)
philipel9b058032020-02-10 11:30:00 +01007564 .WillByDefault(Return(SdpVideoFormat("AV1")));
7565 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007566 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7567 /*allow_default_fallback=*/false));
philipel9b058032020-02-10 11:30:00 +01007568
Henrik Boström381d1092020-05-12 18:49:07 +02007569 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007570 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7571 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7572 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007573 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007574 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007575 /*cwnd_reduce_ratio=*/0);
Markus Handell28c71802021-11-08 10:11:55 +01007576 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007577
7578 video_stream_encoder_->Stop();
7579}
7580
philipel6daa3042022-04-11 10:48:28 +02007581TEST_F(VideoStreamEncoderTest, EncoderSelectorResolutionSwitch) {
7582 NiceMock<MockEncoderSelector> encoder_selector;
7583 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7584 video_send_config_.encoder_settings.encoder_switch_request_callback =
7585 &switch_callback;
7586 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7587 &fake_encoder_, &encoder_selector);
7588 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7589
7590 // Reset encoder for new configuration to take effect.
7591 ConfigureEncoder(video_encoder_config_.Copy());
7592
7593 EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(640, 480)))
7594 .WillOnce(Return(absl::nullopt));
7595 EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(320, 240)))
7596 .WillOnce(Return(SdpVideoFormat("AV1")));
7597 EXPECT_CALL(switch_callback,
7598 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7599 /*allow_default_fallback=*/false));
7600
7601 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7602 /*target_bitrate=*/DataRate::KilobitsPerSec(800),
7603 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(1000),
7604 /*link_allocation=*/DataRate::KilobitsPerSec(1000),
7605 /*fraction_lost=*/0,
7606 /*round_trip_time_ms=*/0,
7607 /*cwnd_reduce_ratio=*/0);
7608
7609 video_source_.IncomingCapturedFrame(CreateFrame(1, 640, 480));
7610 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 480));
7611 video_source_.IncomingCapturedFrame(CreateFrame(3, 320, 240));
7612
7613 AdvanceTime(TimeDelta::Zero());
7614
7615 video_stream_encoder_->Stop();
7616}
7617
philipel9b058032020-02-10 11:30:00 +01007618TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7619 constexpr int kSufficientBitrateToNotDrop = 1000;
7620 constexpr int kDontCare = 100;
7621
7622 NiceMock<MockVideoEncoder> video_encoder;
7623 NiceMock<MockEncoderSelector> encoder_selector;
7624 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7625 video_send_config_.encoder_settings.encoder_switch_request_callback =
7626 &switch_callback;
7627 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7628 &video_encoder, &encoder_selector);
7629 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7630
7631 // Reset encoder for new configuration to take effect.
7632 ConfigureEncoder(video_encoder_config_.Copy());
7633
7634 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7635 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7636 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007637 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007638 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7639 /*stable_target_bitrate=*/
7640 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7641 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007642 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007643 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007644 /*cwnd_reduce_ratio=*/0);
7645
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007646 ON_CALL(video_encoder, Encode)
philipel9b058032020-02-10 11:30:00 +01007647 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007648 ON_CALL(encoder_selector, OnEncoderBroken)
philipel9b058032020-02-10 11:30:00 +01007649 .WillByDefault(Return(SdpVideoFormat("AV2")));
7650
7651 rtc::Event encode_attempted;
7652 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007653 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7654 /*allow_default_fallback=*/true))
7655 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
philipel9b058032020-02-10 11:30:00 +01007656
7657 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7658 encode_attempted.Wait(3000);
7659
Markus Handell28c71802021-11-08 10:11:55 +01007660 AdvanceTime(TimeDelta::Zero());
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007661
philipel9b058032020-02-10 11:30:00 +01007662 video_stream_encoder_->Stop();
7663
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007664 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7665 // to it's factory, so in order for the encoder instance in the
7666 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7667 // reset the `video_stream_encoder_` here.
7668 video_stream_encoder_.reset();
7669}
7670
7671TEST_F(VideoStreamEncoderTest, SwitchEncoderOnInitFailureWithEncoderSelector) {
7672 NiceMock<MockVideoEncoder> video_encoder;
7673 NiceMock<MockEncoderSelector> encoder_selector;
7674 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7675 video_send_config_.encoder_settings.encoder_switch_request_callback =
7676 &switch_callback;
7677 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7678 &video_encoder, &encoder_selector);
7679 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7680
7681 // Reset encoder for new configuration to take effect.
7682 ConfigureEncoder(video_encoder_config_.Copy());
7683
7684 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7685 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7686 /*round_trip_time_ms=*/0,
7687 /*cwnd_reduce_ratio=*/0);
7688 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7689
7690 ON_CALL(video_encoder, InitEncode(_, _))
7691 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7692 ON_CALL(encoder_selector, OnEncoderBroken)
7693 .WillByDefault(Return(SdpVideoFormat("AV2")));
7694
7695 rtc::Event encode_attempted;
7696 EXPECT_CALL(switch_callback,
7697 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7698 /*allow_default_fallback=*/true))
7699 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7700
7701 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7702 encode_attempted.Wait(3000);
7703
7704 AdvanceTime(TimeDelta::Zero());
7705
7706 video_stream_encoder_->Stop();
7707
7708 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7709 // to it's factory, so in order for the encoder instance in the
7710 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7711 // reset the `video_stream_encoder_` here.
7712 video_stream_encoder_.reset();
7713}
7714
7715TEST_F(VideoStreamEncoderTest,
7716 SwitchEncoderOnInitFailureWithoutEncoderSelector) {
7717 NiceMock<MockVideoEncoder> video_encoder;
7718 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7719 video_send_config_.encoder_settings.encoder_switch_request_callback =
7720 &switch_callback;
7721 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7722 &video_encoder, /*encoder_selector=*/nullptr);
7723 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7724
7725 // Reset encoder for new configuration to take effect.
7726 ConfigureEncoder(video_encoder_config_.Copy());
7727
7728 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7729 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7730 /*round_trip_time_ms=*/0,
7731 /*cwnd_reduce_ratio=*/0);
7732 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7733
7734 ON_CALL(video_encoder, InitEncode(_, _))
7735 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7736
7737 rtc::Event encode_attempted;
7738 EXPECT_CALL(switch_callback,
7739 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "VP8"),
7740 /*allow_default_fallback=*/true))
7741 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7742
7743 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7744 encode_attempted.Wait(3000);
7745
7746 AdvanceTime(TimeDelta::Zero());
7747
7748 video_stream_encoder_->Stop();
7749
7750 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007751 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007752 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7753 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007754 video_stream_encoder_.reset();
7755}
7756
Byoungchan Lee13fe3672022-04-06 10:44:42 +09007757TEST_F(VideoStreamEncoderTest, NullEncoderReturnSwitch) {
7758 // As a variant of EncoderSelectorBrokenEncoderSwitch, when a null
7759 // VideoEncoder is passed in encoder_factory, it checks whether
7760 // Codec Switch occurs without a crash.
7761 constexpr int kSufficientBitrateToNotDrop = 1000;
7762 constexpr int kDontCare = 100;
7763
7764 NiceMock<MockEncoderSelector> encoder_selector;
7765 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7766 video_send_config_.encoder_settings.encoder_switch_request_callback =
7767 &switch_callback;
7768 auto encoder_factory =
7769 std::make_unique<test::VideoEncoderNullableProxyFactory>(
7770 /*encoder=*/nullptr, &encoder_selector);
7771 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7772
7773 // Reset encoder for new configuration to take effect.
7774 ConfigureEncoder(video_encoder_config_.Copy());
7775 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7776 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7777 // not fail.
7778 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7779 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7780 /*stable_target_bitrate=*/
7781 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7782 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7783 /*fraction_lost=*/0,
7784 /*round_trip_time_ms=*/0,
7785 /*cwnd_reduce_ratio=*/0);
7786 ON_CALL(encoder_selector, OnEncoderBroken)
7787 .WillByDefault(Return(SdpVideoFormat("AV2")));
7788 rtc::Event encode_attempted;
7789 EXPECT_CALL(switch_callback,
7790 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7791 /*allow_default_fallback=*/_))
7792 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7793
7794 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7795 encode_attempted.Wait(3000);
7796
7797 AdvanceTime(TimeDelta::Zero());
7798
7799 video_stream_encoder_->Stop();
7800
7801 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7802 // to it's factory, so in order for the encoder instance in the
7803 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7804 // reset the `video_stream_encoder_` here.
7805 video_stream_encoder_.reset();
7806}
7807
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007808TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007809 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007810 const int kFrameWidth = 320;
7811 const int kFrameHeight = 180;
7812
7813 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007814 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007815 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007816 /*target_bitrate=*/rate,
7817 /*stable_target_bitrate=*/rate,
7818 /*link_allocation=*/rate,
7819 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007820 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007821 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007822
7823 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007824 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007825 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7826 frame.set_rotation(kVideoRotation_270);
7827 video_source_.IncomingCapturedFrame(frame);
7828 WaitForEncodedFrame(timestamp_ms);
7829 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7830
7831 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007832 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007833 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007834 /*target_bitrate=*/new_stable_rate,
7835 /*stable_target_bitrate=*/new_stable_rate,
7836 /*link_allocation=*/rate,
7837 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007838 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007839 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007840 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7841 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7842 video_stream_encoder_->Stop();
7843}
7844
7845TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007846 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007847 const int kFrameWidth = 320;
7848 const int kFrameHeight = 180;
7849
7850 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007851 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007852 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007853 /*target_bitrate=*/rate,
7854 /*stable_target_bitrate=*/rate,
7855 /*link_allocation=*/rate,
7856 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007857 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007858 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007859
7860 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007861 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007862 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7863 frame.set_rotation(kVideoRotation_270);
7864 video_source_.IncomingCapturedFrame(frame);
7865 WaitForEncodedFrame(timestamp_ms);
7866 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7867
7868 // Set a higher target rate without changing the link_allocation. Should not
7869 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007870 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007871 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007872 /*target_bitrate=*/rate,
7873 /*stable_target_bitrate=*/new_stable_rate,
7874 /*link_allocation=*/rate,
7875 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007876 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007877 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007878 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7879 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7880 video_stream_encoder_->Stop();
7881}
7882
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007883TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01007884 test::ScopedKeyValueConfig field_trials(
7885 field_trials_,
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007886 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7887 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7888 const int kFramerateFps = 30;
7889 const int kWidth = 1920;
7890 const int kHeight = 1080;
7891 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7892 // Works on screenshare mode.
7893 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7894 // We rely on the automatic resolution adaptation, but we handle framerate
7895 // adaptation manually by mocking the stats proxy.
7896 video_source_.set_adaptation_enabled(true);
7897
7898 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007899 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007900 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007901 video_stream_encoder_->SetSource(&video_source_,
7902 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007903 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007904
7905 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7906 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7907
7908 // Pass enough frames with the full update to trigger animation detection.
7909 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007910 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007911 frame.set_ntp_time_ms(timestamp_ms);
7912 frame.set_timestamp_us(timestamp_ms * 1000);
7913 video_source_.IncomingCapturedFrame(frame);
7914 WaitForEncodedFrame(timestamp_ms);
7915 }
7916
7917 // Resolution should be limited.
7918 rtc::VideoSinkWants expected;
7919 expected.max_framerate_fps = kFramerateFps;
7920 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007921 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007922
7923 // Pass one frame with no known update.
7924 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007925 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007926 frame.set_ntp_time_ms(timestamp_ms);
7927 frame.set_timestamp_us(timestamp_ms * 1000);
7928 frame.clear_update_rect();
7929
7930 video_source_.IncomingCapturedFrame(frame);
7931 WaitForEncodedFrame(timestamp_ms);
7932
7933 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007934 EXPECT_THAT(video_source_.sink_wants(),
7935 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007936
7937 video_stream_encoder_->Stop();
7938}
7939
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007940TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7941 const int kWidth = 720; // 540p adapted down.
7942 const int kHeight = 405;
7943 const int kNumFrames = 3;
7944 // Works on screenshare mode.
7945 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7946 /*num_spatial_layers=*/2, /*screenshare=*/true);
7947
7948 video_source_.set_adaptation_enabled(true);
7949
7950 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007951 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007952
7953 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7954
7955 // Pass enough frames with the full update to trigger animation detection.
7956 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007957 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007958 frame.set_ntp_time_ms(timestamp_ms);
7959 frame.set_timestamp_us(timestamp_ms * 1000);
7960 video_source_.IncomingCapturedFrame(frame);
7961 WaitForEncodedFrame(timestamp_ms);
7962 }
7963
7964 video_stream_encoder_->Stop();
7965}
7966
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007967TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7968 const float downscale_factors[] = {4.0, 2.0, 1.0};
7969 const int number_layers =
7970 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7971 VideoEncoderConfig config;
7972 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7973 for (int i = 0; i < number_layers; ++i) {
7974 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7975 config.simulcast_layers[i].active = true;
7976 }
7977 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007978 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007979 "VP8", /*max qp*/ 56, /*screencast*/ false,
7980 /*screenshare enabled*/ false);
7981 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007982 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7983 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007984
7985 // First initialization.
7986 // Encoder should be initialized. Next frame should be key frame.
7987 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7988 sink_.SetNumExpectedLayers(number_layers);
7989 int64_t timestamp_ms = kFrameIntervalMs;
7990 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7991 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007992 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007993 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7994 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7995 VideoFrameType::kVideoFrameKey,
7996 VideoFrameType::kVideoFrameKey}));
7997
7998 // Disable top layer.
7999 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
8000 config.simulcast_layers[number_layers - 1].active = false;
8001 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8002 sink_.SetNumExpectedLayers(number_layers - 1);
8003 timestamp_ms += kFrameIntervalMs;
8004 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8005 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008006 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008007 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8008 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
8009 VideoFrameType::kVideoFrameDelta,
8010 VideoFrameType::kVideoFrameDelta}));
8011
8012 // Re-enable top layer.
8013 // Encoder should be re-initialized. Next frame should be key frame.
8014 config.simulcast_layers[number_layers - 1].active = true;
8015 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8016 sink_.SetNumExpectedLayers(number_layers);
8017 timestamp_ms += kFrameIntervalMs;
8018 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8019 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008020 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008021 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8022 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
8023 VideoFrameType::kVideoFrameKey,
8024 VideoFrameType::kVideoFrameKey}));
8025
8026 // Top layer max rate change.
8027 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
8028 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
8029 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8030 sink_.SetNumExpectedLayers(number_layers);
8031 timestamp_ms += kFrameIntervalMs;
8032 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8033 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008034 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008035 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8036 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
8037 VideoFrameType::kVideoFrameDelta,
8038 VideoFrameType::kVideoFrameDelta}));
8039
8040 // Top layer resolution change.
8041 // Encoder should be re-initialized. Next frame should be key frame.
8042 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
8043 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8044 sink_.SetNumExpectedLayers(number_layers);
8045 timestamp_ms += kFrameIntervalMs;
8046 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8047 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008048 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008049 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8050 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
8051 VideoFrameType::kVideoFrameKey,
8052 VideoFrameType::kVideoFrameKey}));
8053 video_stream_encoder_->Stop();
8054}
8055
Henrik Boström1124ed12021-02-25 10:30:39 +01008056TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
8057 const int kFrameWidth = 1280;
8058 const int kFrameHeight = 720;
8059
8060 SetUp();
8061 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008062 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01008063
8064 // Capturing a frame should reconfigure the encoder and expose the encoder
8065 // resolution, which is the same as the input frame.
8066 int64_t timestamp_ms = kFrameIntervalMs;
8067 video_source_.IncomingCapturedFrame(
8068 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8069 WaitForEncodedFrame(timestamp_ms);
8070 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8071 EXPECT_THAT(video_source_.sink_wants().resolutions,
8072 ::testing::ElementsAreArray(
8073 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
8074
8075 video_stream_encoder_->Stop();
8076}
8077
8078TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
8079 // Pick downscale factors such that we never encode at full resolution - this
8080 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 16:22:11 +02008081 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 10:30:39 +01008082 // encoder should not ask for the frame resolution. This allows video frames
8083 // to have the appearence of one resolution but optimize its internal buffers
8084 // for what is actually encoded.
8085 const size_t kNumSimulcastLayers = 3u;
8086 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
8087 const int kFrameWidth = 1280;
8088 const int kFrameHeight = 720;
8089 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8090 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8091 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8092 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8093 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8094 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8095
8096 VideoEncoderConfig config;
8097 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
8098 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
8099 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
8100 config.simulcast_layers[i].active = true;
8101 }
8102 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02008103 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01008104 "VP8", /*max qp*/ 56, /*screencast*/ false,
8105 /*screenshare enabled*/ false);
8106 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008107 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8108 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01008109
8110 // Capture a frame with all layers active.
8111 int64_t timestamp_ms = kFrameIntervalMs;
8112 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
8113 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8114 video_source_.IncomingCapturedFrame(
8115 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8116 WaitForEncodedFrame(timestamp_ms);
8117 // Expect encoded resolutions to match the expected simulcast layers.
8118 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8119 EXPECT_THAT(
8120 video_source_.sink_wants().resolutions,
8121 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
8122
8123 // Capture a frame with one of the layers inactive.
8124 timestamp_ms += kFrameIntervalMs;
8125 config.simulcast_layers[2].active = false;
8126 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
8127 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8128 video_source_.IncomingCapturedFrame(
8129 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8130 WaitForEncodedFrame(timestamp_ms);
8131
8132 // Expect encoded resolutions to match the expected simulcast layers.
8133 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8134 EXPECT_THAT(video_source_.sink_wants().resolutions,
8135 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
8136
8137 // Capture a frame with all but one layer turned off.
8138 timestamp_ms += kFrameIntervalMs;
8139 config.simulcast_layers[1].active = false;
8140 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
8141 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8142 video_source_.IncomingCapturedFrame(
8143 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8144 WaitForEncodedFrame(timestamp_ms);
8145
8146 // Expect encoded resolutions to match the expected simulcast layers.
8147 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8148 EXPECT_THAT(video_source_.sink_wants().resolutions,
8149 ::testing::ElementsAreArray({kLayer0Size}));
8150
8151 video_stream_encoder_->Stop();
8152}
8153
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008154TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008155 ResetEncoder("VP8", 1, 1, 1, false);
8156
Niels Möller8b692902021-06-14 12:04:57 +02008157 // Force encoder reconfig.
8158 video_source_.IncomingCapturedFrame(
8159 CreateFrame(1, codec_width_, codec_height_));
8160 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8161
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008162 // Set QP on encoded frame and pass the frame to encode complete callback.
8163 // Since QP is present QP parsing won't be triggered and the original value
8164 // should be kept.
8165 EncodedImage encoded_image;
8166 encoded_image.qp_ = 123;
8167 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8168 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8169 CodecSpecificInfo codec_info;
8170 codec_info.codecType = kVideoCodecVP8;
8171 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8172 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8173 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
8174 video_stream_encoder_->Stop();
8175}
8176
8177TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008178 ResetEncoder("VP8", 1, 1, 1, false);
8179
Niels Möller8b692902021-06-14 12:04:57 +02008180 // Force encoder reconfig.
8181 video_source_.IncomingCapturedFrame(
8182 CreateFrame(1, codec_width_, codec_height_));
8183 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8184
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008185 // Pass an encoded frame without QP to encode complete callback. QP should be
8186 // parsed and set.
8187 EncodedImage encoded_image;
8188 encoded_image.qp_ = -1;
8189 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8190 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8191 CodecSpecificInfo codec_info;
8192 codec_info.codecType = kVideoCodecVP8;
8193 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8194 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8195 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
8196 video_stream_encoder_->Stop();
8197}
8198
8199TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01008200 webrtc::test::ScopedKeyValueConfig field_trials(
8201 field_trials_, "WebRTC-QpParsingKillSwitch/Enabled/");
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008202
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008203 ResetEncoder("VP8", 1, 1, 1, false);
8204
Niels Möller8b692902021-06-14 12:04:57 +02008205 // Force encoder reconfig.
8206 video_source_.IncomingCapturedFrame(
8207 CreateFrame(1, codec_width_, codec_height_));
8208 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8209
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008210 EncodedImage encoded_image;
8211 encoded_image.qp_ = -1;
8212 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8213 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8214 CodecSpecificInfo codec_info;
8215 codec_info.codecType = kVideoCodecVP8;
8216 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8217 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8218 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
8219 video_stream_encoder_->Stop();
8220}
8221
Sergey Silkind19e3b92021-03-16 10:05:30 +00008222TEST_F(VideoStreamEncoderTest,
8223 QualityScalingNotAllowed_QualityScalingDisabled) {
8224 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8225
8226 // Disable scaling settings in encoder info.
8227 fake_encoder_.SetQualityScaling(false);
8228 // Disable quality scaling in encoder config.
8229 video_encoder_config.is_quality_scaling_allowed = false;
8230 ConfigureEncoder(std::move(video_encoder_config));
8231
8232 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008233 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008234
8235 test::FrameForwarder source;
8236 video_stream_encoder_->SetSource(
8237 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8238 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8239 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8240
8241 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8242 WaitForEncodedFrame(1);
8243 video_stream_encoder_->TriggerQualityLow();
8244 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8245
8246 video_stream_encoder_->Stop();
8247}
8248
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008249TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
8250 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8251
8252 // Disable scaling settings in encoder info.
8253 fake_encoder_.SetQualityScaling(false);
8254 // Set QP trusted in encoder info.
8255 fake_encoder_.SetIsQpTrusted(true);
8256 // Enable quality scaling in encoder config.
8257 video_encoder_config.is_quality_scaling_allowed = false;
8258 ConfigureEncoder(std::move(video_encoder_config));
8259
8260 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008261 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008262
8263 test::FrameForwarder source;
8264 video_stream_encoder_->SetSource(
8265 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8266 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8267 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8268
8269 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8270 WaitForEncodedFrame(1);
8271 video_stream_encoder_->TriggerQualityLow();
8272 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8273
8274 video_stream_encoder_->Stop();
8275}
8276
Shuhai Pengf2707702021-09-29 17:19:44 +08008277TEST_F(VideoStreamEncoderTest,
8278 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8279 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8280
8281 // Disable scaling settings in encoder info.
8282 fake_encoder_.SetQualityScaling(false);
8283 // Set QP trusted in encoder info.
8284 fake_encoder_.SetIsQpTrusted(true);
8285 // Enable quality scaling in encoder config.
8286 video_encoder_config.is_quality_scaling_allowed = false;
8287 ConfigureEncoder(std::move(video_encoder_config));
8288
8289 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008290 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008291
8292 test::FrameForwarder source;
8293 video_stream_encoder_->SetSource(
8294 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8295 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8296 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8297
8298 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8299 WaitForEncodedFrame(1);
8300 video_stream_encoder_->TriggerQualityLow();
8301 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8302
8303 video_stream_encoder_->Stop();
8304}
8305
8306TEST_F(VideoStreamEncoderTest,
8307 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8308 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8309
8310 // Disable scaling settings in encoder info.
8311 fake_encoder_.SetQualityScaling(false);
8312 // Set QP trusted in encoder info.
8313 fake_encoder_.SetIsQpTrusted(false);
8314 // Enable quality scaling in encoder config.
8315 video_encoder_config.is_quality_scaling_allowed = false;
8316 ConfigureEncoder(std::move(video_encoder_config));
8317
8318 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008319 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008320
8321 test::FrameForwarder source;
8322 video_stream_encoder_->SetSource(
8323 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8324 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8325 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8326
8327 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8328 WaitForEncodedFrame(1);
8329 video_stream_encoder_->TriggerQualityLow();
8330 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8331
8332 video_stream_encoder_->Stop();
8333}
8334
8335TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8336 // Set QP trusted in encoder info.
8337 fake_encoder_.SetIsQpTrusted(false);
8338
8339 const int MinEncBitrateKbps = 30;
8340 const int MaxEncBitrateKbps = 100;
8341 const int MinStartBitrateKbp = 50;
8342 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8343 /*frame_size_pixels=*/codec_width_ * codec_height_,
8344 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8345 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8346 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8347
8348 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008349 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008350
8351 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8352
8353 VideoEncoderConfig video_encoder_config;
8354 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8355 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8356 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8357 MinEncBitrateKbps * 1000;
8358 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8359 kMaxPayloadLength);
8360
8361 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8362 WaitForEncodedFrame(1);
8363 EXPECT_EQ(
8364 MaxEncBitrateKbps,
8365 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8366 EXPECT_EQ(
8367 MinEncBitrateKbps,
8368 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8369
8370 video_stream_encoder_->Stop();
8371}
8372
8373TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8374 // Set QP trusted in encoder info.
8375 fake_encoder_.SetIsQpTrusted(false);
8376
8377 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8378 EncoderInfoSettings::
8379 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8380 codec_width_ * codec_height_,
8381 EncoderInfoSettings::
8382 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8383 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8384
8385 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8386 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8387 const int TargetEncBitrate = MaxEncBitrate;
8388 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8389 DataRate::BitsPerSec(TargetEncBitrate),
8390 DataRate::BitsPerSec(TargetEncBitrate),
8391 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8392
8393 VideoEncoderConfig video_encoder_config;
8394 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8395 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8396 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8397 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8398 kMaxPayloadLength);
8399
8400 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8401 WaitForEncodedFrame(1);
8402 EXPECT_EQ(
8403 MaxEncBitrate / 1000,
8404 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8405 EXPECT_EQ(
8406 MinEncBitrate / 1000,
8407 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8408
8409 video_stream_encoder_->Stop();
8410}
8411
Erik Språnge4589cb2022-04-06 16:44:30 +02008412TEST_F(VideoStreamEncoderTest, NormalComplexityWithMoreThanTwoCores) {
8413 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8414 /*num_spatial_layers=*/1,
8415 /*screenshare=*/false, /*allocation_callback_type=*/
8416 VideoStreamEncoder::BitrateAllocationCallbackType::
8417 kVideoBitrateAllocationWhenScreenSharing,
8418 /*num_cores=*/3);
8419
8420 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8421 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8422 video_source_.IncomingCapturedFrame(
8423 CreateFrame(1, /*width=*/320, /*height=*/180));
8424 WaitForEncodedFrame(1);
8425 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8426 VideoCodecComplexity::kComplexityNormal);
8427 video_stream_encoder_->Stop();
8428}
8429
8430TEST_F(VideoStreamEncoderTest,
8431 NormalComplexityWhenLowTierOptimizationsAreDisabled) {
8432 webrtc::test::ScopedKeyValueConfig field_trials(
8433 field_trials_, "WebRTC-VP9-LowTierOptimizations/Disabled/");
8434
8435 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8436 /*num_spatial_layers=*/1,
8437 /*screenshare=*/false, /*allocation_callback_type=*/
8438 VideoStreamEncoder::BitrateAllocationCallbackType::
8439 kVideoBitrateAllocationWhenScreenSharing,
8440 /*num_cores=*/2);
8441
8442 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8443 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8444 video_source_.IncomingCapturedFrame(
8445 CreateFrame(1, /*width=*/320, /*height=*/180));
8446 WaitForEncodedFrame(1);
8447 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8448 VideoCodecComplexity::kComplexityNormal);
8449 video_stream_encoder_->Stop();
8450}
8451
8452TEST_F(VideoStreamEncoderTest, LowComplexityWithTwoCores) {
8453 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8454 /*num_spatial_layers=*/1,
8455 /*screenshare=*/false, /*allocation_callback_type=*/
8456 VideoStreamEncoder::BitrateAllocationCallbackType::
8457 kVideoBitrateAllocationWhenScreenSharing,
8458 /*num_cores=*/2);
8459
8460 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8461 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8462 video_source_.IncomingCapturedFrame(
8463 CreateFrame(1, /*width=*/320, /*height=*/180));
8464 WaitForEncodedFrame(1);
8465 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8466 VideoCodecComplexity::kComplexityLow);
8467 video_stream_encoder_->Stop();
8468}
8469
Sergey Silkind19e3b92021-03-16 10:05:30 +00008470#if !defined(WEBRTC_IOS)
8471// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8472// disabled by default on iOS.
8473TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8474 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8475
8476 // Disable scaling settings in encoder info.
8477 fake_encoder_.SetQualityScaling(false);
8478 // Enable quality scaling in encoder config.
8479 video_encoder_config.is_quality_scaling_allowed = true;
8480 ConfigureEncoder(std::move(video_encoder_config));
8481
8482 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008483 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008484
8485 test::FrameForwarder source;
8486 video_stream_encoder_->SetSource(
8487 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8488 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8489 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8490
8491 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8492 WaitForEncodedFrame(1);
8493 video_stream_encoder_->TriggerQualityLow();
8494 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8495
8496 video_stream_encoder_->Stop();
8497}
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008498
8499TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8500 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8501
8502 // Disable scaling settings in encoder info.
8503 fake_encoder_.SetQualityScaling(false);
8504 // Set QP trusted in encoder info.
8505 fake_encoder_.SetIsQpTrusted(true);
8506 // Enable quality scaling in encoder config.
8507 video_encoder_config.is_quality_scaling_allowed = true;
8508 ConfigureEncoder(std::move(video_encoder_config));
8509
8510 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008511 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008512
8513 test::FrameForwarder source;
8514 video_stream_encoder_->SetSource(
8515 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8516 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8517 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8518
8519 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8520 WaitForEncodedFrame(1);
8521 video_stream_encoder_->TriggerQualityLow();
8522 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8523
8524 video_stream_encoder_->Stop();
8525}
Shuhai Pengf2707702021-09-29 17:19:44 +08008526
8527TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8528 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8529
8530 // Disable scaling settings in encoder info.
8531 fake_encoder_.SetQualityScaling(false);
8532 // Set QP not trusted in encoder info.
8533 fake_encoder_.SetIsQpTrusted(false);
8534 // Enable quality scaling in encoder config.
8535 video_encoder_config.is_quality_scaling_allowed = true;
8536 ConfigureEncoder(std::move(video_encoder_config));
8537
8538 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008539 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008540
8541 test::FrameForwarder source;
8542 video_stream_encoder_->SetSource(
8543 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8544 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8545 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8546
8547 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8548 WaitForEncodedFrame(1);
8549 video_stream_encoder_->TriggerQualityLow();
8550 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8551 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8552 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8553
8554 video_stream_encoder_->Stop();
8555}
8556
8557TEST_F(VideoStreamEncoderTest,
8558 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8559 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8560
8561 // Disable scaling settings in encoder info.
8562 fake_encoder_.SetQualityScaling(false);
8563 // Set QP trusted in encoder info.
8564 fake_encoder_.SetIsQpTrusted(true);
8565 // Enable quality scaling in encoder config.
8566 video_encoder_config.is_quality_scaling_allowed = true;
8567 ConfigureEncoder(std::move(video_encoder_config));
8568
8569 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008570 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008571
8572 test::FrameForwarder source;
8573 video_stream_encoder_->SetSource(
8574 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8575 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8576 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8577
8578 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8579 WaitForEncodedFrame(1);
8580 video_stream_encoder_->TriggerQualityLow();
8581 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8582 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8583
8584 video_stream_encoder_->Stop();
8585}
8586
8587TEST_F(VideoStreamEncoderTest,
8588 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8589 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8590
8591 // Disable scaling settings in encoder info.
8592 fake_encoder_.SetQualityScaling(false);
8593 // Set QP trusted in encoder info.
8594 fake_encoder_.SetIsQpTrusted(false);
8595 // Enable quality scaling in encoder config.
8596 video_encoder_config.is_quality_scaling_allowed = true;
8597 ConfigureEncoder(std::move(video_encoder_config));
8598
8599 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008600 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008601
8602 test::FrameForwarder source;
8603 video_stream_encoder_->SetSource(
8604 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8605 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8606 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8607
8608 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8609 WaitForEncodedFrame(1);
8610 video_stream_encoder_->TriggerQualityLow();
8611 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8612
8613 video_stream_encoder_->Stop();
8614}
8615
Erik Språng5e13d052022-08-02 11:42:49 +02008616TEST_F(VideoStreamEncoderTest,
8617 RequestsRefreshFrameAfterEarlyDroppedNativeFrame) {
8618 // Send a native frame before encoder rates have been set. The encoder is
8619 // seen as paused at this time.
8620 rtc::Event frame_destroyed_event;
8621 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
8622 /*ntp_time_ms=*/1, &frame_destroyed_event, codec_width_, codec_height_));
8623
8624 // Frame should be dropped and destroyed.
8625 ExpectDroppedFrame();
8626 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
8627 EXPECT_EQ(video_source_.refresh_frames_requested_, 0);
8628
8629 // Set bitrates, unpausing the encoder and triggering a request for a refresh
8630 // frame.
8631 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8632 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8633 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8634 EXPECT_EQ(video_source_.refresh_frames_requested_, 1);
8635
8636 video_stream_encoder_->Stop();
8637}
8638
Erik Språnge4589cb2022-04-06 16:44:30 +02008639#endif // !defined(WEBRTC_IOS)
Sergey Silkind19e3b92021-03-16 10:05:30 +00008640
Henrik Boström56db9ff2021-03-24 09:06:45 +01008641// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8642class VideoStreamEncoderWithRealEncoderTest
8643 : public VideoStreamEncoderTest,
8644 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8645 public:
8646 VideoStreamEncoderWithRealEncoderTest()
8647 : VideoStreamEncoderTest(),
8648 codec_type_(std::get<0>(GetParam())),
8649 allow_i420_conversion_(std::get<1>(GetParam())) {}
8650
8651 void SetUp() override {
8652 VideoStreamEncoderTest::SetUp();
8653 std::unique_ptr<VideoEncoder> encoder;
8654 switch (codec_type_) {
8655 case kVideoCodecVP8:
8656 encoder = VP8Encoder::Create();
8657 break;
8658 case kVideoCodecVP9:
8659 encoder = VP9Encoder::Create();
8660 break;
8661 case kVideoCodecAV1:
philipel09a28482022-05-25 09:47:06 +02008662 encoder = CreateLibaomAv1Encoder();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008663 break;
8664 case kVideoCodecH264:
8665 encoder =
8666 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8667 break;
8668 case kVideoCodecMultiplex:
8669 mock_encoder_factory_for_multiplex_ =
8670 std::make_unique<MockVideoEncoderFactory>();
8671 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8672 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8673 .WillRepeatedly([] { return VP8Encoder::Create(); });
8674 encoder = std::make_unique<MultiplexEncoderAdapter>(
8675 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8676 false);
8677 break;
8678 default:
Artem Titovd3251962021-11-15 16:57:07 +01008679 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008680 }
8681 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8682 }
8683
8684 void TearDown() override {
8685 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 16:22:11 +02008686 // Ensure `video_stream_encoder_` is destroyed before
8687 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 09:06:45 +01008688 video_stream_encoder_.reset();
8689 VideoStreamEncoderTest::TearDown();
8690 }
8691
8692 protected:
8693 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8694 std::unique_ptr<VideoEncoder> encoder) {
8695 // Configure VSE to use the encoder.
8696 encoder_ = std::move(encoder);
8697 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8698 encoder_.get(), &encoder_selector_);
8699 video_send_config_.encoder_settings.encoder_factory =
8700 encoder_proxy_factory_.get();
8701 VideoEncoderConfig video_encoder_config;
8702 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8703 video_encoder_config_ = video_encoder_config.Copy();
8704 ConfigureEncoder(video_encoder_config_.Copy());
8705
8706 // Set bitrate to ensure frame is not dropped.
8707 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008708 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008709 }
8710
8711 const VideoCodecType codec_type_;
8712 const bool allow_i420_conversion_;
8713 NiceMock<MockEncoderSelector> encoder_selector_;
8714 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8715 std::unique_ptr<VideoEncoder> encoder_;
8716 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8717};
8718
8719TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8720 auto native_i420_frame = test::CreateMappableNativeFrame(
8721 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8722 video_source_.IncomingCapturedFrame(native_i420_frame);
8723 WaitForEncodedFrame(codec_width_, codec_height_);
8724
8725 auto mappable_native_buffer =
8726 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8727 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8728 mappable_native_buffer->GetMappedFramedBuffers();
8729 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8730 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8731 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8732 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8733}
8734
8735TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8736 auto native_nv12_frame = test::CreateMappableNativeFrame(
8737 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8738 video_source_.IncomingCapturedFrame(native_nv12_frame);
8739 WaitForEncodedFrame(codec_width_, codec_height_);
8740
8741 auto mappable_native_buffer =
8742 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8743 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8744 mappable_native_buffer->GetMappedFramedBuffers();
8745 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8746 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8747 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8748 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8749
8750 if (!allow_i420_conversion_) {
8751 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8752 }
8753}
8754
Erik Språng7444b192021-06-02 14:02:13 +02008755TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8756 if (codec_type_ == kVideoCodecMultiplex) {
8757 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8758 return;
8759 }
8760
8761 const size_t kNumSpatialLayers = 3u;
8762 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8763 const int kFrameWidth = 1280;
8764 const int kFrameHeight = 720;
8765 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8766 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8767 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8768 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8769 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8770 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8771
8772 VideoEncoderConfig config;
8773 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8774 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008775 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008776 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8777 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8778 vp9_settings.numberOfTemporalLayers = 3;
8779 vp9_settings.automaticResizeOn = false;
8780 config.encoder_specific_settings =
8781 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8782 vp9_settings);
8783 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8784 /*fps=*/30.0,
8785 /*first_active_layer=*/0,
8786 /*num_spatial_layers=*/3,
8787 /*num_temporal_layers=*/3,
8788 /*is_screenshare=*/false);
8789 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8790 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008791 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008792 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8793 /*fps=*/30.0,
8794 /*first_active_layer=*/0,
8795 /*num_spatial_layers=*/3,
8796 /*num_temporal_layers=*/3,
8797 /*is_screenshare=*/false);
Niels Möller79d566b2022-04-29 11:03:13 +02008798 config.simulcast_layers[0].scalability_mode = ScalabilityMode::kL3T3_KEY;
Erik Språng7444b192021-06-02 14:02:13 +02008799 } else {
8800 // Simulcast for VP8/H264.
8801 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8802 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8803 config.simulcast_layers[i].scale_resolution_down_by =
8804 kDownscaleFactors[i];
8805 config.simulcast_layers[i].active = true;
8806 }
8807 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8808 // Turn off frame dropping to prevent flakiness.
Niels Möller807328f2022-05-12 16:16:39 +02008809 config.frame_drop_enabled = false;
Erik Språng7444b192021-06-02 14:02:13 +02008810 }
8811 }
8812
8813 auto set_layer_active = [&](int layer_idx, bool active) {
8814 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8815 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8816 config.spatial_layers[layer_idx].active = active;
8817 } else {
8818 config.simulcast_layers[layer_idx].active = active;
8819 }
8820 };
8821
8822 config.video_stream_factory =
8823 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8824 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8825 /*screencast*/ false,
8826 /*screenshare enabled*/ false);
8827 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008828 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8829 0, 0, 0);
Erik Språng7444b192021-06-02 14:02:13 +02008830
8831 // Capture a frame with all layers active.
8832 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8833 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8834 int64_t timestamp_ms = kFrameIntervalMs;
8835 video_source_.IncomingCapturedFrame(
8836 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8837
8838 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8839 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8840
8841 // Capture a frame with one of the layers inactive.
8842 set_layer_active(2, false);
8843 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8844 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8845 timestamp_ms += kFrameIntervalMs;
8846 video_source_.IncomingCapturedFrame(
8847 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8848 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8849
8850 // New target bitrates signaled based on lower resolution.
8851 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8852 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8853 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8854 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8855
8856 // Re-enable the top layer.
8857 set_layer_active(2, true);
8858 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8859 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8860 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8861
8862 // Bitrate target adjusted back up to enable HD layer...
8863 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8864 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8865 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8866 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8867
8868 // ...then add a new frame.
8869 timestamp_ms += kFrameIntervalMs;
8870 video_source_.IncomingCapturedFrame(
8871 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8872 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8873 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8874
8875 video_stream_encoder_->Stop();
8876}
8877
Henrik Boström56db9ff2021-03-24 09:06:45 +01008878std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8879 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8880 VideoCodecType codec_type = std::get<0>(info.param);
8881 bool allow_i420_conversion = std::get<1>(info.param);
8882 std::string str;
8883 switch (codec_type) {
8884 case kVideoCodecGeneric:
8885 str = "Generic";
8886 break;
8887 case kVideoCodecVP8:
8888 str = "VP8";
8889 break;
8890 case kVideoCodecVP9:
8891 str = "VP9";
8892 break;
8893 case kVideoCodecAV1:
8894 str = "AV1";
8895 break;
8896 case kVideoCodecH264:
8897 str = "H264";
8898 break;
8899 case kVideoCodecMultiplex:
8900 str = "Multiplex";
8901 break;
8902 default:
Artem Titovd3251962021-11-15 16:57:07 +01008903 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008904 }
8905 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8906 return str;
8907}
8908
8909constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8910 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8911constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8912 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8913constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
Ilya Nikolaevskiy1bcdafc2022-03-09 16:01:07 +01008914 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/false);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008915constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8916 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8917#if defined(WEBRTC_USE_H264)
8918constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8919 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8920
8921// The windows compiler does not tolerate #if statements inside the
8922// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8923// and without H264).
8924INSTANTIATE_TEST_SUITE_P(
8925 All,
8926 VideoStreamEncoderWithRealEncoderTest,
8927 ::testing::Values(kVP8DisallowConversion,
8928 kVP9DisallowConversion,
8929 kAV1AllowConversion,
8930 kMultiplexDisallowConversion,
8931 kH264AllowConversion),
8932 TestParametersVideoCodecAndAllowI420ConversionToString);
8933#else
8934INSTANTIATE_TEST_SUITE_P(
8935 All,
8936 VideoStreamEncoderWithRealEncoderTest,
8937 ::testing::Values(kVP8DisallowConversion,
8938 kVP9DisallowConversion,
8939 kAV1AllowConversion,
8940 kMultiplexDisallowConversion),
8941 TestParametersVideoCodecAndAllowI420ConversionToString);
8942#endif
8943
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008944class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8945 protected:
8946 void RunTest(const std::vector<VideoStream>& configs,
8947 const int expected_num_init_encode) {
8948 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008949 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008950 InsertFrameAndWaitForEncoded();
8951 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8952 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008953 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8954 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008955
8956 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8957 ConfigureEncoder(configs[1]);
8958 InsertFrameAndWaitForEncoded();
8959 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8960 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008961 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008962 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 10:07:11 +02008963 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008964
8965 video_stream_encoder_->Stop();
8966 }
8967
8968 void ConfigureEncoder(const VideoStream& stream) {
8969 VideoEncoderConfig config;
8970 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8971 config.max_bitrate_bps = stream.max_bitrate_bps;
8972 config.simulcast_layers[0] = stream;
8973 config.video_stream_factory =
8974 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8975 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8976 /*conference_mode=*/false);
8977 video_stream_encoder_->ConfigureEncoder(std::move(config),
8978 kMaxPayloadLength);
8979 }
8980
8981 void OnBitrateUpdated(DataRate bitrate) {
8982 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8983 bitrate, bitrate, bitrate, 0, 0, 0);
8984 }
8985
8986 void InsertFrameAndWaitForEncoded() {
8987 timestamp_ms_ += kFrameIntervalMs;
8988 video_source_.IncomingCapturedFrame(
8989 CreateFrame(timestamp_ms_, kWidth, kHeight));
8990 sink_.WaitForEncodedFrame(timestamp_ms_);
8991 }
8992
8993 void ExpectEqual(const VideoCodec& actual,
8994 const VideoStream& expected) const {
8995 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8996 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8997 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
8998 static_cast<unsigned int>(expected.min_bitrate_bps));
8999 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
9000 static_cast<unsigned int>(expected.max_bitrate_bps));
9001 EXPECT_EQ(actual.simulcastStream[0].width,
9002 kWidth / expected.scale_resolution_down_by);
9003 EXPECT_EQ(actual.simulcastStream[0].height,
9004 kHeight / expected.scale_resolution_down_by);
9005 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
9006 expected.num_temporal_layers);
Niels Möller79d566b2022-04-29 11:03:13 +02009007 EXPECT_EQ(actual.GetScalabilityMode(), expected.scalability_mode);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02009008 }
9009
9010 VideoStream DefaultConfig() const {
9011 VideoStream stream;
9012 stream.max_framerate = 25;
9013 stream.min_bitrate_bps = 35000;
9014 stream.max_bitrate_bps = 900000;
9015 stream.scale_resolution_down_by = 1.0;
9016 stream.num_temporal_layers = 1;
9017 stream.bitrate_priority = 1.0;
Niels Möller79d566b2022-04-29 11:03:13 +02009018 stream.scalability_mode = absl::nullopt;
Åsa Persson4d4f62f2021-09-12 16:14:48 +02009019 return stream;
9020 }
9021
9022 const int kWidth = 640;
9023 const int kHeight = 360;
9024 int64_t timestamp_ms_ = 0;
9025};
9026
9027TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
9028 VideoStream config1 = DefaultConfig();
9029 VideoStream config2 = config1;
9030 config2.max_framerate++;
9031
9032 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9033}
9034
9035TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
9036 VideoStream config1 = DefaultConfig();
9037 VideoStream config2 = config1;
9038 config2.min_bitrate_bps += 10000;
9039
9040 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9041}
9042
9043TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
9044 VideoStream config1 = DefaultConfig();
9045 VideoStream config2 = config1;
9046 config2.max_bitrate_bps += 100000;
9047
9048 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9049}
9050
9051TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
9052 VideoStream config1 = DefaultConfig();
9053 VideoStream config2 = config1;
9054 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
9055
9056 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9057}
9058
9059TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
9060 VideoStream config1 = DefaultConfig();
9061 VideoStream config2 = config1;
9062 config2.scale_resolution_down_by *= 2;
9063
9064 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9065}
9066
9067TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
9068 VideoStream config1 = DefaultConfig();
9069 VideoStream config2 = config1;
9070 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
9071
9072 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9073}
9074
9075TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
9076 VideoStream config1 = DefaultConfig();
9077 VideoStream config2 = config1;
Niels Möller05954892022-05-13 13:34:37 +02009078 config2.scalability_mode = ScalabilityMode::kL2T1;
9079
9080 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9081}
9082
9083TEST_F(ReconfigureEncoderTest,
9084 UpdatesNumTemporalLayersFromScalabilityModeChanges) {
9085 VideoStream config1 = DefaultConfig();
9086 VideoStream config2 = config1;
Niels Möller79d566b2022-04-29 11:03:13 +02009087 config2.scalability_mode = ScalabilityMode::kL1T2;
Niels Möller05954892022-05-13 13:34:37 +02009088 config2.num_temporal_layers = 2;
Åsa Persson4d4f62f2021-09-12 16:14:48 +02009089
9090 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9091}
9092
Tommi62b01db2022-01-25 23:41:22 +01009093// Simple test that just creates and then immediately destroys an encoder.
9094// The purpose of the test is to make sure that nothing bad happens if the
9095// initialization step on the encoder queue, doesn't run.
9096TEST(VideoStreamEncoderSimpleTest, CreateDestroy) {
9097 class SuperLazyTaskQueue : public webrtc::TaskQueueBase {
9098 public:
9099 SuperLazyTaskQueue() = default;
9100 ~SuperLazyTaskQueue() override = default;
9101
9102 private:
9103 void Delete() override { delete this; }
Danil Chapovalov95eeaa72022-07-06 10:14:29 +02009104 void PostTask(absl::AnyInvocable<void() &&> task) override {
Tommi62b01db2022-01-25 23:41:22 +01009105 // meh.
9106 }
Danil Chapovalov95eeaa72022-07-06 10:14:29 +02009107 void PostDelayedTask(absl::AnyInvocable<void() &&> task,
9108 TimeDelta delay) override {
Tommi62b01db2022-01-25 23:41:22 +01009109 ASSERT_TRUE(false);
9110 }
Danil Chapovalov95eeaa72022-07-06 10:14:29 +02009111 void PostDelayedHighPrecisionTask(absl::AnyInvocable<void() &&> task,
9112 TimeDelta delay) override {
9113 ADD_FAILURE();
9114 }
Tommi62b01db2022-01-25 23:41:22 +01009115 };
9116
9117 // Lots of boiler plate.
Jonas Oreland8ca06132022-03-14 12:52:48 +01009118 test::ScopedKeyValueConfig field_trials;
Tommi62b01db2022-01-25 23:41:22 +01009119 GlobalSimulatedTimeController time_controller(Timestamp::Millis(0));
9120 auto stats_proxy = std::make_unique<MockableSendStatisticsProxy>(
9121 time_controller.GetClock(), VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 12:52:48 +01009122 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo, field_trials);
Tommi62b01db2022-01-25 23:41:22 +01009123 SimpleVideoStreamEncoderFactory::MockFakeEncoder mock_fake_encoder(
9124 time_controller.GetClock());
9125 test::VideoEncoderProxyFactory encoder_factory(&mock_fake_encoder);
9126 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory =
9127 CreateBuiltinVideoBitrateAllocatorFactory();
9128 VideoStreamEncoderSettings encoder_settings{
9129 VideoEncoder::Capabilities(/*loss_notification=*/false)};
9130 encoder_settings.encoder_factory = &encoder_factory;
9131 encoder_settings.bitrate_allocator_factory = bitrate_allocator_factory.get();
9132
9133 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9134 EXPECT_CALL((*adapter.get()), Initialize).WillOnce(Return());
9135
9136 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
9137 encoder_queue(new SuperLazyTaskQueue());
9138
9139 // Construct a VideoStreamEncoder instance and let it go out of scope without
9140 // doing anything else (including calling Stop()). This should be fine since
9141 // the posted init task will simply be deleted.
9142 auto encoder = std::make_unique<VideoStreamEncoder>(
9143 time_controller.GetClock(), 1, stats_proxy.get(), encoder_settings,
Markus Handell8e4197b2022-05-30 15:45:28 +02009144 std::make_unique<CpuOveruseDetectorProxy>(stats_proxy.get(),
9145 field_trials),
Tommi62b01db2022-01-25 23:41:22 +01009146 std::move(adapter), std::move(encoder_queue),
9147 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009148 kVideoBitrateAllocation,
9149 field_trials);
Tommi4d6dd172022-05-23 09:34:41 +02009150
9151 // Stop the encoder explicitly. This additional step tests if we could
9152 // hang when calling stop and the TQ has been stopped and/or isn't accepting
9153 // any more tasks.
9154 encoder->Stop();
Tommi62b01db2022-01-25 23:41:22 +01009155}
9156
Markus Handellb4e96d42021-11-05 12:00:55 +01009157TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
9158 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9159 auto* adapter_ptr = adapter.get();
9160 SimpleVideoStreamEncoderFactory factory;
Markus Handell8d87c462021-12-16 11:37:16 +01009161 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9162 nullptr;
9163 EXPECT_CALL(*adapter_ptr, Initialize)
9164 .WillOnce(Invoke([&video_stream_encoder_callback](
9165 FrameCadenceAdapterInterface::Callback* callback) {
9166 video_stream_encoder_callback = callback;
9167 }));
9168 TaskQueueBase* encoder_queue = nullptr;
9169 auto video_stream_encoder =
9170 factory.Create(std::move(adapter), &encoder_queue);
Markus Handellb4e96d42021-11-05 12:00:55 +01009171
Markus Handelle59fee82021-12-23 09:29:23 +01009172 // First a call before we know the frame size and hence cannot compute the
9173 // number of simulcast layers.
9174 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
9175 &FrameCadenceAdapterInterface::
9176 ZeroHertzModeParams::num_simulcast_layers,
9177 Eq(0)))));
Markus Handellb4e96d42021-11-05 12:00:55 +01009178 VideoEncoderConfig config;
Markus Handell8d87c462021-12-16 11:37:16 +01009179 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
Markus Handellb4e96d42021-11-05 12:00:55 +01009180 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9181 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
Markus Handelle59fee82021-12-23 09:29:23 +01009182 factory.DepleteTaskQueues();
9183
9184 // Then a call as we've computed the number of simulcast layers after a passed
9185 // frame.
9186 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
9187 &FrameCadenceAdapterInterface::
9188 ZeroHertzModeParams::num_simulcast_layers,
9189 Gt(0)))));
Markus Handell8d87c462021-12-16 11:37:16 +01009190 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handell9a478b52021-11-18 16:07:01 +01009191 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01009192 Mock::VerifyAndClearExpectations(adapter_ptr);
9193
Markus Handelle59fee82021-12-23 09:29:23 +01009194 // Expect a disabled zero-hertz mode after passing realtime video.
Markus Handell8d87c462021-12-16 11:37:16 +01009195 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Eq(absl::nullopt)));
Markus Handellb4e96d42021-11-05 12:00:55 +01009196 VideoEncoderConfig config2;
Markus Handell8d87c462021-12-16 11:37:16 +01009197 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config2);
Markus Handellb4e96d42021-11-05 12:00:55 +01009198 config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
9199 video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
Markus Handell8d87c462021-12-16 11:37:16 +01009200 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
Markus Handell9a478b52021-11-18 16:07:01 +01009201 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01009202}
9203
9204TEST(VideoStreamEncoderFrameCadenceTest,
9205 ForwardsFramesIntoFrameCadenceAdapter) {
9206 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9207 auto* adapter_ptr = adapter.get();
9208 test::FrameForwarder video_source;
9209 SimpleVideoStreamEncoderFactory factory;
9210 auto video_stream_encoder = factory.Create(std::move(adapter));
9211 video_stream_encoder->SetSource(
9212 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9213
9214 EXPECT_CALL(*adapter_ptr, OnFrame);
9215 auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
9216 video_source.IncomingCapturedFrame(
Markus Handell8d87c462021-12-16 11:37:16 +01009217 VideoFrame::Builder().set_video_frame_buffer(std::move(buffer)).build());
Markus Handellb4e96d42021-11-05 12:00:55 +01009218}
9219
Markus Handellee225432021-11-29 12:35:12 +01009220TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) {
9221 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9222 auto* adapter_ptr = adapter.get();
9223 test::FrameForwarder video_source;
9224 SimpleVideoStreamEncoderFactory factory;
9225 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9226 nullptr;
9227 EXPECT_CALL(*adapter_ptr, Initialize)
9228 .WillOnce(Invoke([&video_stream_encoder_callback](
9229 FrameCadenceAdapterInterface::Callback* callback) {
9230 video_stream_encoder_callback = callback;
9231 }));
9232 TaskQueueBase* encoder_queue = nullptr;
9233 auto video_stream_encoder =
9234 factory.Create(std::move(adapter), &encoder_queue);
9235
9236 // This is just to make the VSE operational. We'll feed a frame directly by
9237 // the callback interface.
9238 video_stream_encoder->SetSource(
9239 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9240
9241 VideoEncoderConfig video_encoder_config;
9242 test::FillEncoderConfiguration(kVideoCodecGeneric, 1, &video_encoder_config);
9243 video_stream_encoder->ConfigureEncoder(std::move(video_encoder_config),
9244 /*max_data_payload_length=*/1000);
9245
9246 EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps);
9247 EXPECT_CALL(*adapter_ptr, UpdateFrameRate);
Markus Handell8d87c462021-12-16 11:37:16 +01009248 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handellee225432021-11-29 12:35:12 +01009249 factory.DepleteTaskQueues();
9250}
9251
Markus Handell8d87c462021-12-16 11:37:16 +01009252TEST(VideoStreamEncoderFrameCadenceTest,
9253 DeactivatesActivatesLayersOnBitrateChanges) {
9254 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9255 auto* adapter_ptr = adapter.get();
9256 SimpleVideoStreamEncoderFactory factory;
9257 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9258 nullptr;
9259 EXPECT_CALL(*adapter_ptr, Initialize)
9260 .WillOnce(Invoke([&video_stream_encoder_callback](
9261 FrameCadenceAdapterInterface::Callback* callback) {
9262 video_stream_encoder_callback = callback;
9263 }));
9264 TaskQueueBase* encoder_queue = nullptr;
9265 auto video_stream_encoder =
9266 factory.Create(std::move(adapter), &encoder_queue);
9267
9268 // Configure 2 simulcast layers. FillEncoderConfiguration sets min bitrates to
9269 // {150000, 450000}.
9270 VideoEncoderConfig video_encoder_config;
9271 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9272 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9273 kMaxPayloadLength);
9274 // Ensure an encoder is created.
9275 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9276
9277 // Both layers enabled at 1 MBit/s.
9278 video_stream_encoder->OnBitrateUpdated(
9279 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9280 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9281 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9282 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9283 factory.DepleteTaskQueues();
9284 Mock::VerifyAndClearExpectations(adapter_ptr);
9285
9286 // Layer 1 disabled at 200 KBit/s.
9287 video_stream_encoder->OnBitrateUpdated(
9288 DataRate::KilobitsPerSec(200), DataRate::KilobitsPerSec(200),
9289 DataRate::KilobitsPerSec(200), 0, 0, 0);
9290 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9291 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9292 factory.DepleteTaskQueues();
9293 Mock::VerifyAndClearExpectations(adapter_ptr);
9294
9295 // All layers off at suspended video.
9296 video_stream_encoder->OnBitrateUpdated(DataRate::Zero(), DataRate::Zero(),
9297 DataRate::Zero(), 0, 0, 0);
9298 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/false));
9299 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9300 factory.DepleteTaskQueues();
9301 Mock::VerifyAndClearExpectations(adapter_ptr);
9302
9303 // Both layers enabled again back at 1 MBit/s.
9304 video_stream_encoder->OnBitrateUpdated(
9305 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9306 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9307 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9308 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9309 factory.DepleteTaskQueues();
9310}
9311
9312TEST(VideoStreamEncoderFrameCadenceTest, UpdatesQualityConvergence) {
9313 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9314 auto* adapter_ptr = adapter.get();
9315 SimpleVideoStreamEncoderFactory factory;
9316 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9317 nullptr;
9318 EXPECT_CALL(*adapter_ptr, Initialize)
9319 .WillOnce(Invoke([&video_stream_encoder_callback](
9320 FrameCadenceAdapterInterface::Callback* callback) {
9321 video_stream_encoder_callback = callback;
9322 }));
9323 TaskQueueBase* encoder_queue = nullptr;
9324 auto video_stream_encoder =
9325 factory.Create(std::move(adapter), &encoder_queue);
9326
9327 // Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder.
9328 VideoEncoderConfig video_encoder_config;
9329 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9330 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9331 kMaxPayloadLength);
9332 video_stream_encoder->OnBitrateUpdated(
9333 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9334 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9335
9336 // Pass a frame which has unconverged results.
9337 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9338 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9339 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9340 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
9341 EXPECT_FALSE(encoded_image.IsAtTargetQuality());
9342 CodecSpecificInfo codec_specific;
9343 codec_specific.codecType = kVideoCodecGeneric;
9344 return codec_specific;
9345 }));
9346 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, false));
9347 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9348 factory.DepleteTaskQueues();
9349 Mock::VerifyAndClearExpectations(adapter_ptr);
9350 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9351
9352 // Pass a frame which converges in layer 0 and not in layer 1.
9353 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9354 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9355 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9356 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
9357 encoded_image.SetAtTargetQuality(encoded_image.SpatialIndex() == 0);
9358 CodecSpecificInfo codec_specific;
9359 codec_specific.codecType = kVideoCodecGeneric;
9360 return codec_specific;
9361 }));
9362 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, true));
9363 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9364 factory.DepleteTaskQueues();
9365 Mock::VerifyAndClearExpectations(adapter_ptr);
9366 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9367}
9368
Markus Handell2e0f4f02021-12-21 19:14:58 +01009369TEST(VideoStreamEncoderFrameCadenceTest,
9370 RequestsRefreshFramesWhenCadenceAdapterInstructs) {
9371 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9372 auto* adapter_ptr = adapter.get();
9373 MockVideoSourceInterface mock_source;
9374 SimpleVideoStreamEncoderFactory factory;
9375 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9376 nullptr;
9377 EXPECT_CALL(*adapter_ptr, Initialize)
9378 .WillOnce(Invoke([&video_stream_encoder_callback](
9379 FrameCadenceAdapterInterface::Callback* callback) {
9380 video_stream_encoder_callback = callback;
9381 }));
9382 TaskQueueBase* encoder_queue = nullptr;
9383 auto video_stream_encoder =
9384 factory.Create(std::move(adapter), &encoder_queue);
9385 video_stream_encoder->SetSource(
9386 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9387 VideoEncoderConfig config;
9388 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9389 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9390 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9391 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9392 // Ensure the encoder is set up.
9393 factory.DepleteTaskQueues();
9394
Markus Handell818e7fb2021-12-30 13:01:33 +01009395 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest)
9396 .WillOnce(Invoke([video_stream_encoder_callback] {
9397 video_stream_encoder_callback->RequestRefreshFrame();
9398 }));
Markus Handell2e0f4f02021-12-21 19:14:58 +01009399 EXPECT_CALL(mock_source, RequestRefreshFrame);
9400 video_stream_encoder->SendKeyFrame();
9401 factory.DepleteTaskQueues();
9402 Mock::VerifyAndClearExpectations(adapter_ptr);
9403 Mock::VerifyAndClearExpectations(&mock_source);
9404
Markus Handell818e7fb2021-12-30 13:01:33 +01009405 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest);
Markus Handell2e0f4f02021-12-21 19:14:58 +01009406 EXPECT_CALL(mock_source, RequestRefreshFrame).Times(0);
9407 video_stream_encoder->SendKeyFrame();
9408 factory.DepleteTaskQueues();
9409}
9410
Markus Handell818e7fb2021-12-30 13:01:33 +01009411TEST(VideoStreamEncoderFrameCadenceTest,
9412 RequestsRefreshFrameForEarlyZeroHertzKeyFrameRequest) {
9413 SimpleVideoStreamEncoderFactory factory;
9414 auto encoder_queue =
9415 factory.GetTimeController()->GetTaskQueueFactory()->CreateTaskQueue(
9416 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
9417
9418 // Enables zero-hertz mode.
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009419 test::ScopedKeyValueConfig field_trials(
9420 "WebRTC-ZeroHertzScreenshare/Enabled/");
Markus Handell818e7fb2021-12-30 13:01:33 +01009421 auto adapter = FrameCadenceAdapterInterface::Create(
Jonas Oreland8ca06132022-03-14 12:52:48 +01009422 factory.GetTimeController()->GetClock(), encoder_queue.get(),
9423 field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009424 FrameCadenceAdapterInterface* adapter_ptr = adapter.get();
9425
9426 MockVideoSourceInterface mock_source;
9427 auto video_stream_encoder = factory.CreateWithEncoderQueue(
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009428 std::move(adapter), std::move(encoder_queue), &field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009429
9430 video_stream_encoder->SetSource(
9431 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9432 VideoEncoderConfig config;
9433 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9434 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9435 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9436
9437 // Eventually expect a refresh frame request when requesting a key frame
9438 // before initializing zero-hertz mode. This can happen in reality because the
9439 // threads invoking key frame requests and constraints setup aren't
9440 // synchronized.
9441 EXPECT_CALL(mock_source, RequestRefreshFrame);
9442 video_stream_encoder->SendKeyFrame();
Markus Handellf5a50792022-06-16 14:25:15 +02009443 constexpr int kMaxFps = 30;
9444 adapter_ptr->OnConstraintsChanged(VideoTrackSourceConstraints{0, kMaxFps});
9445 factory.GetTimeController()->AdvanceTime(
9446 TimeDelta::Seconds(1) *
9447 FrameCadenceAdapterInterface::kOnDiscardedFrameRefreshFramePeriod /
9448 kMaxFps);
Markus Handell818e7fb2021-12-30 13:01:33 +01009449}
9450
perkj26091b12016-09-01 01:17:40 -07009451} // namespace webrtc