blob: 3a5b03d1db9305560358392a65dc599b792121bc [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Markus Handell8e4197b2022-05-30 15:45:28 +020020#include "api/field_trials_view.h"
Markus Handell9a478b52021-11-18 16:07:01 +010021#include "api/rtp_parameters.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020022#include "api/task_queue/default_task_queue_factory.h"
Markus Handell818e7fb2021-12-30 13:01:33 +010023#include "api/task_queue/task_queue_base.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010024#include "api/task_queue/task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020025#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010026#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010027#include "api/test/mock_video_encoder_factory.h"
Markus Handell8d87c462021-12-16 11:37:16 +010028#include "api/units/data_rate.h"
Markus Handellf5a50792022-06-16 14:25:15 +020029#include "api/units/time_delta.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080030#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020031#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020032#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020033#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010034#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010035#include "api/video_codecs/sdp_video_format.h"
Erik Språnge4589cb2022-04-06 16:44:30 +020036#include "api/video_codecs/video_codec.h"
Elad Alon370f93a2019-06-11 14:57:57 +020037#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020038#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010039#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020040#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010041#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020042#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070043#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080044#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020045#include "media/engine/webrtc_video_engine.h"
philipel09a28482022-05-25 09:47:06 +020046#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010047#include "modules/video_coding/codecs/h264/include/h264.h"
48#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
49#include "modules/video_coding/codecs/vp8/include/vp8.h"
50#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020051#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020052#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020053#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010054#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020055#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010056#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020057#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020058#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080059#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020060#include "rtc_base/synchronization/mutex.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020061#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020062#include "test/encoder_settings.h"
63#include "test/fake_encoder.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010064#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020065#include "test/gmock.h"
66#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010067#include "test/mappable_native_buffer.h"
Jonas Orelandc7f691a2022-03-09 15:12:07 +010068#include "test/scoped_key_value_config.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020069#include "test/time_controller/simulated_time_controller.h"
Byoungchan Lee13fe3672022-04-06 10:44:42 +090070#include "test/video_encoder_nullable_proxy_factory.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020071#include "test/video_encoder_proxy_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010072#include "video/frame_cadence_adapter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020073#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070074
75namespace webrtc {
76
sprang57c2fff2017-01-16 06:24:02 -080077using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020078using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020079using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020080using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020081using ::testing::Ge;
82using ::testing::Gt;
Markus Handellee225432021-11-29 12:35:12 +010083using ::testing::Invoke;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020084using ::testing::Le;
85using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010086using ::testing::Matcher;
Markus Handellb4e96d42021-11-05 12:00:55 +010087using ::testing::Mock;
philipel9b058032020-02-10 11:30:00 +010088using ::testing::NiceMock;
Markus Handellb4e96d42021-11-05 12:00:55 +010089using ::testing::Optional;
philipel9b058032020-02-10 11:30:00 +010090using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010091using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020092using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080093
perkj803d97f2016-11-01 11:45:46 -070094namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020095const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010096const int kQpLow = 1;
97const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020098const int kMinFramerateFps = 2;
99const int kMinBalancedFramerateFps = 7;
Markus Handell2cfc1af2022-08-19 08:16:48 +0000100constexpr TimeDelta kFrameTimeout = TimeDelta::Millis(100);
asapersson5f7226f2016-11-25 04:37:00 -0800101const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 10:07:11 +0200102const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
103const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
104const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
105const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 07:02:22 -0800106const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -0700107const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200108const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +0200109const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200110const VideoEncoder::ResolutionBitrateLimits
111 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
112const VideoEncoder::ResolutionBitrateLimits
113 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800114
Asa Persson606d3cb2021-10-04 10:07:11 +0200115uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200116 0x00, 0x00, 0x03, 0x03, 0xF4,
117 0x05, 0x03, 0xC7, 0xE0, 0x1B,
118 0x41, 0x10, 0x8D, 0x00};
119
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100120const uint8_t kCodedFrameVp8Qp25[] = {
121 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
122 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
123 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
124
Markus Handell818e7fb2021-12-30 13:01:33 +0100125VideoFrame CreateSimpleNV12Frame() {
126 return VideoFrame::Builder()
127 .set_video_frame_buffer(rtc::make_ref_counted<NV12Buffer>(
128 /*width=*/16, /*height=*/16))
129 .build();
130}
131
Markus Handell8d87c462021-12-16 11:37:16 +0100132void PassAFrame(
133 TaskQueueBase* encoder_queue,
134 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback,
135 int64_t ntp_time_ms) {
Danil Chapovalov95eeaa72022-07-06 10:14:29 +0200136 encoder_queue->PostTask([video_stream_encoder_callback, ntp_time_ms] {
137 video_stream_encoder_callback->OnFrame(Timestamp::Millis(ntp_time_ms), 1,
138 CreateSimpleNV12Frame());
139 });
Markus Handell8d87c462021-12-16 11:37:16 +0100140}
141
perkj803d97f2016-11-01 11:45:46 -0700142class TestBuffer : public webrtc::I420Buffer {
143 public:
144 TestBuffer(rtc::Event* event, int width, int height)
145 : I420Buffer(width, height), event_(event) {}
146
147 private:
148 friend class rtc::RefCountedObject<TestBuffer>;
149 ~TestBuffer() override {
150 if (event_)
151 event_->Set();
152 }
153 rtc::Event* const event_;
154};
155
Henrik Boström56db9ff2021-03-24 09:06:45 +0100156// A fake native buffer that can't be converted to I420. Upon scaling, it
157// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700158class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
159 public:
160 FakeNativeBuffer(rtc::Event* event, int width, int height)
161 : event_(event), width_(width), height_(height) {}
162 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
163 int width() const override { return width_; }
164 int height() const override { return height_; }
165 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
166 return nullptr;
167 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100168 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
169 int offset_x,
170 int offset_y,
171 int crop_width,
172 int crop_height,
173 int scaled_width,
174 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200175 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
176 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100177 }
Noah Richards51db4212019-06-12 06:59:12 -0700178
179 private:
180 friend class rtc::RefCountedObject<FakeNativeBuffer>;
181 ~FakeNativeBuffer() override {
182 if (event_)
183 event_->Set();
184 }
185 rtc::Event* const event_;
186 const int width_;
187 const int height_;
188};
189
Evan Shrubsole895556e2020-10-05 09:15:13 +0200190// A fake native buffer that is backed by an NV12 buffer.
191class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
192 public:
193 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
194 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
195
196 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
197 int width() const override { return nv12_buffer_->width(); }
198 int height() const override { return nv12_buffer_->height(); }
199 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
200 return nv12_buffer_->ToI420();
201 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200202 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
203 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
204 if (absl::c_find(types, Type::kNV12) != types.end()) {
205 return nv12_buffer_;
206 }
207 return nullptr;
208 }
Niels Möllerba2de582022-04-20 16:46:26 +0200209 const NV12BufferInterface* GetNV12() const { return nv12_buffer_.get(); }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200210
211 private:
212 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
213 ~FakeNV12NativeBuffer() override {
214 if (event_)
215 event_->Set();
216 }
217 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
218 rtc::Event* const event_;
219};
220
Niels Möller7dc26b72017-12-06 10:27:48 +0100221class CpuOveruseDetectorProxy : public OveruseFrameDetector {
222 public:
Markus Handell8e4197b2022-05-30 15:45:28 +0200223 CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer,
224 const FieldTrialsView& field_trials)
225 : OveruseFrameDetector(metrics_observer, field_trials),
Henrik Boström381d1092020-05-12 18:49:07 +0200226 last_target_framerate_fps_(-1),
227 framerate_updated_event_(true /* manual_reset */,
228 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100229 virtual ~CpuOveruseDetectorProxy() {}
230
231 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200232 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100233 last_target_framerate_fps_ = framerate_fps;
234 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200235 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100236 }
237
238 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200239 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100240 return last_target_framerate_fps_;
241 }
242
Niels Möller4db138e2018-04-19 09:04:13 +0200243 CpuOveruseOptions GetOptions() { return options_; }
244
Henrik Boström381d1092020-05-12 18:49:07 +0200245 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
246
Niels Möller7dc26b72017-12-06 10:27:48 +0100247 private:
Markus Handella3765182020-07-08 13:13:32 +0200248 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100249 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200250 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100251};
252
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200253class FakeVideoSourceRestrictionsListener
254 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200255 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200256 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200257 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200258 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200259 RTC_DCHECK(was_restrictions_updated_);
260 }
261
262 rtc::Event* restrictions_updated_event() {
263 return &restrictions_updated_event_;
264 }
265
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200266 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200267 void OnVideoSourceRestrictionsUpdated(
268 VideoSourceRestrictions restrictions,
269 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200270 rtc::scoped_refptr<Resource> reason,
271 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200272 was_restrictions_updated_ = true;
273 restrictions_updated_event_.Set();
274 }
275
276 private:
277 bool was_restrictions_updated_;
278 rtc::Event restrictions_updated_event_;
279};
280
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200281auto WantsFps(Matcher<int> fps_matcher) {
282 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
283 fps_matcher);
284}
285
286auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
287 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
288 AllOf(max_pixel_matcher, Gt(0)));
289}
290
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200291auto ResolutionMax() {
292 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200293 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200294 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
295 Eq(absl::nullopt)));
296}
297
298auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200299 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200300}
301
302auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200303 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200304}
305
306auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200307 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200308}
309
310auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200311 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200312}
313
314auto FpsMaxResolutionMax() {
315 return AllOf(FpsMax(), ResolutionMax());
316}
317
318auto UnlimitedSinkWants() {
319 return AllOf(FpsUnlimited(), ResolutionMax());
320}
321
322auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
323 Matcher<int> fps_range_matcher;
324
325 if (last_frame_pixels <= 320 * 240) {
326 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200327 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200328 fps_range_matcher = AllOf(Ge(10), Le(15));
329 } else if (last_frame_pixels <= 640 * 480) {
330 fps_range_matcher = Ge(15);
331 } else {
332 fps_range_matcher = Eq(kDefaultFramerate);
333 }
334 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
335 fps_range_matcher);
336}
337
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200338auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
339 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
340 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
341}
342
343auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
344 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
345}
346
347auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
348 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
349}
350
351auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
352 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
353 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
354}
355
356auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
357 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
358 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
359}
360
361auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
362 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
363 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
364}
365
366auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
367 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
368 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
369}
370
mflodmancc3d4422017-08-03 08:27:51 -0700371class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700372 public:
Markus Handell9a478b52021-11-18 16:07:01 +0100373 VideoStreamEncoderUnderTest(
374 TimeController* time_controller,
375 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter,
376 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
377 encoder_queue,
378 SendStatisticsProxy* stats_proxy,
379 const VideoStreamEncoderSettings& settings,
380 VideoStreamEncoder::BitrateAllocationCallbackType
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100381 allocation_callback_type,
Erik Språnge4589cb2022-04-06 16:44:30 +0200382 const FieldTrialsView& field_trials,
383 int num_cores)
Markus Handell8e4197b2022-05-30 15:45:28 +0200384 : VideoStreamEncoder(
385 time_controller->GetClock(),
386 num_cores,
387 stats_proxy,
388 settings,
389 std::unique_ptr<OveruseFrameDetector>(
390 overuse_detector_proxy_ =
391 new CpuOveruseDetectorProxy(stats_proxy, field_trials)),
392 std::move(cadence_adapter),
393 std::move(encoder_queue),
394 allocation_callback_type,
395 field_trials),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200396 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200397 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200398 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200399 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200400 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200401 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200402 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200403 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100404 }
perkj803d97f2016-11-01 11:45:46 -0700405
Henrik Boström381d1092020-05-12 18:49:07 +0200406 void SetSourceAndWaitForRestrictionsUpdated(
407 rtc::VideoSourceInterface<VideoFrame>* source,
408 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200409 FakeVideoSourceRestrictionsListener listener;
410 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200411 SetSource(source, degradation_preference);
Markus Handell2cfc1af2022-08-19 08:16:48 +0000412 listener.restrictions_updated_event()->Wait(TimeDelta::Seconds(5));
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200413 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200414 }
415
416 void SetSourceAndWaitForFramerateUpdated(
417 rtc::VideoSourceInterface<VideoFrame>* source,
418 const DegradationPreference& degradation_preference) {
419 overuse_detector_proxy_->framerate_updated_event()->Reset();
420 SetSource(source, degradation_preference);
Markus Handell2cfc1af2022-08-19 08:16:48 +0000421 overuse_detector_proxy_->framerate_updated_event()->Wait(
422 TimeDelta::Seconds(5));
Henrik Boström381d1092020-05-12 18:49:07 +0200423 }
424
425 void OnBitrateUpdatedAndWaitForManagedResources(
426 DataRate target_bitrate,
427 DataRate stable_target_bitrate,
428 DataRate link_allocation,
429 uint8_t fraction_lost,
430 int64_t round_trip_time_ms,
431 double cwnd_reduce_ratio) {
432 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
433 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
434 // Bitrate is updated on the encoder queue.
435 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200436 }
437
kthelgason2fc52542017-03-03 00:24:41 -0800438 // This is used as a synchronisation mechanism, to make sure that the
439 // encoder queue is not blocked before we start sending it frames.
440 void WaitUntilTaskQueueIsIdle() {
Markus Handell28c71802021-11-08 10:11:55 +0100441 time_controller_->AdvanceTime(TimeDelta::Zero());
kthelgason2fc52542017-03-03 00:24:41 -0800442 }
443
Henrik Boström91aa7322020-04-28 12:24:33 +0200444 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200445 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200446 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200447 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200448 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200449 event.Set();
450 });
Markus Handell2cfc1af2022-08-19 08:16:48 +0000451 ASSERT_TRUE(event.Wait(TimeDelta::Seconds(5)));
Markus Handell28c71802021-11-08 10:11:55 +0100452 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200453 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200454
Henrik Boström91aa7322020-04-28 12:24:33 +0200455 void TriggerCpuUnderuse() {
456 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200457 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200458 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200459 event.Set();
460 });
Markus Handell2cfc1af2022-08-19 08:16:48 +0000461 ASSERT_TRUE(event.Wait(TimeDelta::Seconds(5)));
Markus Handell28c71802021-11-08 10:11:55 +0100462 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200463 }
kthelgason876222f2016-11-29 01:44:11 -0800464
Henrik Boström91aa7322020-04-28 12:24:33 +0200465 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200466 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200467 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200468 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200469 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200470 event.Set();
471 });
Markus Handell2cfc1af2022-08-19 08:16:48 +0000472 ASSERT_TRUE(event.Wait(TimeDelta::Seconds(5)));
Markus Handell28c71802021-11-08 10:11:55 +0100473 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200474 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200475 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200476 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200477 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200478 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200479 event.Set();
480 });
Markus Handell2cfc1af2022-08-19 08:16:48 +0000481 ASSERT_TRUE(event.Wait(TimeDelta::Seconds(5)));
Markus Handell28c71802021-11-08 10:11:55 +0100482 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200483 }
484
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200485 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100486 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200487 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
488 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200489 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700490};
491
Noah Richards51db4212019-06-12 06:59:12 -0700492// Simulates simulcast behavior and makes highest stream resolutions divisible
493// by 4.
494class CroppingVideoStreamFactory
495 : public VideoEncoderConfig::VideoStreamFactoryInterface {
496 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200497 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700498
499 private:
500 std::vector<VideoStream> CreateEncoderStreams(
501 int width,
502 int height,
503 const VideoEncoderConfig& encoder_config) override {
504 std::vector<VideoStream> streams = test::CreateVideoStreams(
505 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700506 return streams;
507 }
Noah Richards51db4212019-06-12 06:59:12 -0700508};
509
sprangb1ca0732017-02-01 08:38:12 -0800510class AdaptingFrameForwarder : public test::FrameForwarder {
511 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200512 explicit AdaptingFrameForwarder(TimeController* time_controller)
513 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700514 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800515
516 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200517 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800518 adaptation_enabled_ = enabled;
519 }
520
asaperssonfab67072017-04-04 05:51:49 -0700521 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200522 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800523 return adaptation_enabled_;
524 }
525
Henrik Boström1124ed12021-02-25 10:30:39 +0100526 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
527 // the resolution or frame rate was different than it is currently. If
528 // something else is modified, such as encoder resolutions, but the resolution
529 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700530 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200531 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700532 return last_wants_;
533 }
534
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200535 absl::optional<int> last_sent_width() const { return last_width_; }
536 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800537
sprangb1ca0732017-02-01 08:38:12 -0800538 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200539 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +0100540 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200541
sprangb1ca0732017-02-01 08:38:12 -0800542 int cropped_width = 0;
543 int cropped_height = 0;
544 int out_width = 0;
545 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700546 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000547 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
548 << "w=" << video_frame.width()
549 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700550 if (adapter_.AdaptFrameResolution(
551 video_frame.width(), video_frame.height(),
552 video_frame.timestamp_us() * 1000, &cropped_width,
553 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100554 VideoFrame adapted_frame =
555 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200556 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100557 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200558 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100559 .set_timestamp_ms(99)
560 .set_rotation(kVideoRotation_0)
561 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100562 if (video_frame.has_update_rect()) {
563 adapted_frame.set_update_rect(
564 video_frame.update_rect().ScaleWithFrame(
565 video_frame.width(), video_frame.height(), 0, 0,
566 video_frame.width(), video_frame.height(), out_width,
567 out_height));
568 }
sprangc5d62e22017-04-02 23:53:04 -0700569 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800570 last_width_.emplace(adapted_frame.width());
571 last_height_.emplace(adapted_frame.height());
572 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200573 last_width_ = absl::nullopt;
574 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700575 }
sprangb1ca0732017-02-01 08:38:12 -0800576 } else {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000577 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800578 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800579 last_width_.emplace(video_frame.width());
580 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800581 }
582 }
583
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200584 void OnOutputFormatRequest(int width, int height) {
585 absl::optional<std::pair<int, int>> target_aspect_ratio =
586 std::make_pair(width, height);
587 absl::optional<int> max_pixel_count = width * height;
588 absl::optional<int> max_fps;
589 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
590 max_fps);
591 }
592
sprangb1ca0732017-02-01 08:38:12 -0800593 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
594 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200595 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100596 rtc::VideoSinkWants prev_wants = sink_wants_locked();
597 bool did_adapt =
598 prev_wants.max_pixel_count != wants.max_pixel_count ||
599 prev_wants.target_pixel_count != wants.target_pixel_count ||
600 prev_wants.max_framerate_fps != wants.max_framerate_fps;
601 if (did_adapt) {
602 last_wants_ = prev_wants;
603 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100604 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200605 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800606 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200607
Erik Språng5e13d052022-08-02 11:42:49 +0200608 void RequestRefreshFrame() override { ++refresh_frames_requested_; }
609
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200610 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800611 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200612 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
613 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200614 absl::optional<int> last_width_;
615 absl::optional<int> last_height_;
Erik Språng5e13d052022-08-02 11:42:49 +0200616 int refresh_frames_requested_{0};
sprangb1ca0732017-02-01 08:38:12 -0800617};
sprangc5d62e22017-04-02 23:53:04 -0700618
Niels Möller213618e2018-07-24 09:29:58 +0200619// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700620class MockableSendStatisticsProxy : public SendStatisticsProxy {
621 public:
622 MockableSendStatisticsProxy(Clock* clock,
623 const VideoSendStream::Config& config,
Jonas Oreland8ca06132022-03-14 12:52:48 +0100624 VideoEncoderConfig::ContentType content_type,
Jonas Orelande62c2f22022-03-29 11:04:48 +0200625 const FieldTrialsView& field_trials)
Jonas Oreland8ca06132022-03-14 12:52:48 +0100626 : SendStatisticsProxy(clock, config, content_type, field_trials) {}
sprangc5d62e22017-04-02 23:53:04 -0700627
628 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200629 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700630 if (mock_stats_)
631 return *mock_stats_;
632 return SendStatisticsProxy::GetStats();
633 }
634
Niels Möller213618e2018-07-24 09:29:58 +0200635 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200636 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200637 if (mock_stats_)
638 return mock_stats_->input_frame_rate;
639 return SendStatisticsProxy::GetInputFrameRate();
640 }
sprangc5d62e22017-04-02 23:53:04 -0700641 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200642 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700643 mock_stats_.emplace(stats);
644 }
645
646 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200647 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700648 mock_stats_.reset();
649 }
650
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200651 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
652 on_frame_dropped_ = std::move(callback);
653 }
654
sprangc5d62e22017-04-02 23:53:04 -0700655 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200656 void OnFrameDropped(DropReason reason) override {
657 SendStatisticsProxy::OnFrameDropped(reason);
658 if (on_frame_dropped_)
659 on_frame_dropped_(reason);
660 }
661
Markus Handella3765182020-07-08 13:13:32 +0200662 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200663 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200664 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700665};
666
Markus Handellb4e96d42021-11-05 12:00:55 +0100667class SimpleVideoStreamEncoderFactory {
668 public:
669 class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
670 public:
671 using VideoStreamEncoder::VideoStreamEncoder;
672 ~AdaptedVideoStreamEncoder() { Stop(); }
673 };
674
Markus Handell8d87c462021-12-16 11:37:16 +0100675 class MockFakeEncoder : public test::FakeEncoder {
676 public:
677 using FakeEncoder::FakeEncoder;
678 MOCK_METHOD(CodecSpecificInfo,
679 EncodeHook,
680 (EncodedImage & encoded_image,
681 rtc::scoped_refptr<EncodedImageBuffer> buffer),
682 (override));
683 };
684
Markus Handellee225432021-11-29 12:35:12 +0100685 SimpleVideoStreamEncoderFactory() {
Markus Handellb4e96d42021-11-05 12:00:55 +0100686 encoder_settings_.encoder_factory = &encoder_factory_;
Markus Handellee225432021-11-29 12:35:12 +0100687 encoder_settings_.bitrate_allocator_factory =
688 bitrate_allocator_factory_.get();
Markus Handellb4e96d42021-11-05 12:00:55 +0100689 }
690
Markus Handell818e7fb2021-12-30 13:01:33 +0100691 std::unique_ptr<AdaptedVideoStreamEncoder> CreateWithEncoderQueue(
Markus Handellee225432021-11-29 12:35:12 +0100692 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100693 std::unique_ptr<TaskQueueBase, TaskQueueDeleter> encoder_queue,
Jonas Orelande62c2f22022-03-29 11:04:48 +0200694 const FieldTrialsView* field_trials = nullptr) {
Markus Handellb4e96d42021-11-05 12:00:55 +0100695 auto result = std::make_unique<AdaptedVideoStreamEncoder>(
696 time_controller_.GetClock(),
697 /*number_of_cores=*/1,
698 /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
Markus Handell8e4197b2022-05-30 15:45:28 +0200699 std::make_unique<CpuOveruseDetectorProxy>(
700 /*stats_proxy=*/nullptr,
701 field_trials ? *field_trials : field_trials_),
Markus Handellee225432021-11-29 12:35:12 +0100702 std::move(zero_hertz_adapter), std::move(encoder_queue),
Markus Handellb4e96d42021-11-05 12:00:55 +0100703 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100704 kVideoBitrateAllocation,
705 field_trials ? *field_trials : field_trials_);
Markus Handellb4e96d42021-11-05 12:00:55 +0100706 result->SetSink(&sink_, /*rotation_applied=*/false);
707 return result;
708 }
709
Markus Handell818e7fb2021-12-30 13:01:33 +0100710 std::unique_ptr<AdaptedVideoStreamEncoder> Create(
711 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
712 TaskQueueBase** encoder_queue_ptr = nullptr) {
713 auto encoder_queue =
714 time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
715 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
716 if (encoder_queue_ptr)
717 *encoder_queue_ptr = encoder_queue.get();
718 return CreateWithEncoderQueue(std::move(zero_hertz_adapter),
719 std::move(encoder_queue));
720 }
721
Markus Handell9a478b52021-11-18 16:07:01 +0100722 void DepleteTaskQueues() { time_controller_.AdvanceTime(TimeDelta::Zero()); }
Markus Handell8d87c462021-12-16 11:37:16 +0100723 MockFakeEncoder& GetMockFakeEncoder() { return mock_fake_encoder_; }
Markus Handell9a478b52021-11-18 16:07:01 +0100724
Markus Handell818e7fb2021-12-30 13:01:33 +0100725 GlobalSimulatedTimeController* GetTimeController() {
726 return &time_controller_;
727 }
728
Markus Handellb4e96d42021-11-05 12:00:55 +0100729 private:
730 class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
731 public:
732 ~NullEncoderSink() override = default;
733 void OnEncoderConfigurationChanged(
734 std::vector<VideoStream> streams,
735 bool is_svc,
736 VideoEncoderConfig::ContentType content_type,
737 int min_transmit_bitrate_bps) override {}
738 void OnBitrateAllocationUpdated(
739 const VideoBitrateAllocation& allocation) override {}
740 void OnVideoLayersAllocationUpdated(
741 VideoLayersAllocation allocation) override {}
742 Result OnEncodedImage(
743 const EncodedImage& encoded_image,
744 const CodecSpecificInfo* codec_specific_info) override {
745 return Result(EncodedImageCallback::Result::OK);
746 }
747 };
748
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100749 test::ScopedKeyValueConfig field_trials_;
Philipp Hanckea204ad22022-07-08 18:43:25 +0200750 GlobalSimulatedTimeController time_controller_{Timestamp::Zero()};
Markus Handellee225432021-11-29 12:35:12 +0100751 std::unique_ptr<TaskQueueFactory> task_queue_factory_{
752 time_controller_.CreateTaskQueueFactory()};
753 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_ =
754 std::make_unique<MockableSendStatisticsProxy>(
755 time_controller_.GetClock(),
756 VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 12:52:48 +0100757 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
758 field_trials_);
Markus Handellee225432021-11-29 12:35:12 +0100759 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_ =
760 CreateBuiltinVideoBitrateAllocatorFactory();
761 VideoStreamEncoderSettings encoder_settings_{
762 VideoEncoder::Capabilities(/*loss_notification=*/false)};
Markus Handell8d87c462021-12-16 11:37:16 +0100763 MockFakeEncoder mock_fake_encoder_{time_controller_.GetClock()};
764 test::VideoEncoderProxyFactory encoder_factory_{&mock_fake_encoder_};
Markus Handellb4e96d42021-11-05 12:00:55 +0100765 NullEncoderSink sink_;
766};
767
768class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
769 public:
770 MOCK_METHOD(void, Initialize, (Callback * callback), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100771 MOCK_METHOD(void,
772 SetZeroHertzModeEnabled,
773 (absl::optional<ZeroHertzModeParams>),
774 (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100775 MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
Markus Handellee225432021-11-29 12:35:12 +0100776 MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override));
777 MOCK_METHOD(void, UpdateFrameRate, (), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100778 MOCK_METHOD(void,
779 UpdateLayerQualityConvergence,
Markus Handell5a77e512022-09-01 12:51:50 +0000780 (size_t spatial_index, bool converged),
Markus Handell8d87c462021-12-16 11:37:16 +0100781 (override));
782 MOCK_METHOD(void,
783 UpdateLayerStatus,
Markus Handell5a77e512022-09-01 12:51:50 +0000784 (size_t spatial_index, bool enabled),
Markus Handell8d87c462021-12-16 11:37:16 +0100785 (override));
Markus Handell818e7fb2021-12-30 13:01:33 +0100786 MOCK_METHOD(void, ProcessKeyFrameRequest, (), (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100787};
788
philipel9b058032020-02-10 11:30:00 +0100789class MockEncoderSelector
790 : public VideoEncoderFactory::EncoderSelectorInterface {
791 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200792 MOCK_METHOD(void,
793 OnCurrentEncoder,
794 (const SdpVideoFormat& format),
795 (override));
796 MOCK_METHOD(absl::optional<SdpVideoFormat>,
797 OnAvailableBitrate,
798 (const DataRate& rate),
799 (override));
philipel6daa3042022-04-11 10:48:28 +0200800 MOCK_METHOD(absl::optional<SdpVideoFormat>,
801 OnResolutionChange,
802 (const RenderResolution& resolution),
803 (override));
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200804 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100805};
806
Markus Handell2e0f4f02021-12-21 19:14:58 +0100807class MockVideoSourceInterface : public rtc::VideoSourceInterface<VideoFrame> {
808 public:
809 MOCK_METHOD(void,
810 AddOrUpdateSink,
811 (rtc::VideoSinkInterface<VideoFrame>*,
812 const rtc::VideoSinkWants&),
813 (override));
814 MOCK_METHOD(void,
815 RemoveSink,
816 (rtc::VideoSinkInterface<VideoFrame>*),
817 (override));
818 MOCK_METHOD(void, RequestRefreshFrame, (), (override));
819};
820
perkj803d97f2016-11-01 11:45:46 -0700821} // namespace
822
mflodmancc3d4422017-08-03 08:27:51 -0700823class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700824 public:
Markus Handell2cfc1af2022-08-19 08:16:48 +0000825 static constexpr TimeDelta kDefaultTimeout = TimeDelta::Seconds(1);
perkj26091b12016-09-01 01:17:40 -0700826
mflodmancc3d4422017-08-03 08:27:51 -0700827 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700828 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700829 codec_width_(320),
830 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200831 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200832 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200833 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700834 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200835 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700836 video_send_config_,
Jonas Oreland8ca06132022-03-14 12:52:48 +0100837 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
838 field_trials_)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200839 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700840
841 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700842 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700843 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200844 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800845 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200846 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200847 video_send_config_.rtp.payload_name = "FAKE";
848 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700849
Per512ecb32016-09-23 15:52:06 +0200850 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200851 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200852 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
853 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
854 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100855 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700856
Niels Möllerf1338562018-04-26 09:51:47 +0200857 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800858 }
859
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100860 void ConfigureEncoder(
861 VideoEncoderConfig video_encoder_config,
862 VideoStreamEncoder::BitrateAllocationCallbackType
863 allocation_callback_type =
864 VideoStreamEncoder::BitrateAllocationCallbackType::
Erik Språnge4589cb2022-04-06 16:44:30 +0200865 kVideoBitrateAllocationWhenScreenSharing,
866 int num_cores = 1) {
mflodmancc3d4422017-08-03 08:27:51 -0700867 if (video_stream_encoder_)
868 video_stream_encoder_->Stop();
Markus Handell9a478b52021-11-18 16:07:01 +0100869
870 auto encoder_queue = GetTaskQueueFactory()->CreateTaskQueue(
871 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
872 TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
873 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter =
874 FrameCadenceAdapterInterface::Create(time_controller_.GetClock(),
Jonas Oreland8ca06132022-03-14 12:52:48 +0100875 encoder_queue_ptr, field_trials_);
Markus Handell9a478b52021-11-18 16:07:01 +0100876 video_stream_encoder_ = std::make_unique<VideoStreamEncoderUnderTest>(
877 &time_controller_, std::move(cadence_adapter), std::move(encoder_queue),
878 stats_proxy_.get(), video_send_config_.encoder_settings,
Erik Språnge4589cb2022-04-06 16:44:30 +0200879 allocation_callback_type, field_trials_, num_cores);
Asa Persson606d3cb2021-10-04 10:07:11 +0200880 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 08:27:51 -0700881 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700882 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 10:07:11 +0200883 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 08:27:51 -0700884 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200885 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700886 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800887 }
888
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100889 void ResetEncoder(const std::string& payload_name,
890 size_t num_streams,
891 size_t num_temporal_layers,
892 unsigned char num_spatial_layers,
893 bool screenshare,
894 VideoStreamEncoder::BitrateAllocationCallbackType
895 allocation_callback_type =
896 VideoStreamEncoder::BitrateAllocationCallbackType::
Erik Språnge4589cb2022-04-06 16:44:30 +0200897 kVideoBitrateAllocationWhenScreenSharing,
898 int num_cores = 1) {
Niels Möller259a4972018-04-05 15:36:51 +0200899 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800900
901 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200902 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
903 num_streams, &video_encoder_config);
904 for (auto& layer : video_encoder_config.simulcast_layers) {
905 layer.num_temporal_layers = num_temporal_layers;
906 layer.max_framerate = kDefaultFramerate;
907 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100908 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 10:07:11 +0200909 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 07:06:52 -0700910 video_encoder_config.content_type =
911 screenshare ? VideoEncoderConfig::ContentType::kScreen
912 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700913 if (payload_name == "VP9") {
914 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
915 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200916 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700917 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200918 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
919 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700920 }
Erik Språnge4589cb2022-04-06 16:44:30 +0200921 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type,
922 num_cores);
perkj26091b12016-09-01 01:17:40 -0700923 }
924
sprang57c2fff2017-01-16 06:24:02 -0800925 VideoFrame CreateFrame(int64_t ntp_time_ms,
926 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200927 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200928 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200929 destruction_event, codec_width_, codec_height_))
930 .set_ntp_time_ms(ntp_time_ms)
931 .set_timestamp_ms(99)
932 .set_rotation(kVideoRotation_0)
933 .build();
perkj26091b12016-09-01 01:17:40 -0700934 }
935
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100936 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
937 rtc::Event* destruction_event,
938 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200939 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200940 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200941 destruction_event, codec_width_, codec_height_))
942 .set_ntp_time_ms(ntp_time_ms)
943 .set_timestamp_ms(99)
944 .set_rotation(kVideoRotation_0)
945 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
946 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100947 }
948
sprang57c2fff2017-01-16 06:24:02 -0800949 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200950 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
951 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200952 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200953 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200954 .set_ntp_time_ms(ntp_time_ms)
955 .set_timestamp_ms(ntp_time_ms)
956 .set_rotation(kVideoRotation_0)
957 .build();
perkj803d97f2016-11-01 11:45:46 -0700958 }
959
Evan Shrubsole895556e2020-10-05 09:15:13 +0200960 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200961 return VideoFrame::Builder()
962 .set_video_frame_buffer(NV12Buffer::Create(width, height))
963 .set_ntp_time_ms(ntp_time_ms)
964 .set_timestamp_ms(ntp_time_ms)
965 .set_rotation(kVideoRotation_0)
966 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200967 }
968
Noah Richards51db4212019-06-12 06:59:12 -0700969 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
970 rtc::Event* destruction_event,
971 int width,
972 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200973 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200974 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200975 destruction_event, width, height))
976 .set_ntp_time_ms(ntp_time_ms)
977 .set_timestamp_ms(99)
978 .set_rotation(kVideoRotation_0)
979 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700980 }
981
Evan Shrubsole895556e2020-10-05 09:15:13 +0200982 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
983 rtc::Event* destruction_event,
984 int width,
985 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200986 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200987 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200988 destruction_event, width, height))
989 .set_ntp_time_ms(ntp_time_ms)
990 .set_timestamp_ms(99)
991 .set_rotation(kVideoRotation_0)
992 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200993 }
994
Noah Richards51db4212019-06-12 06:59:12 -0700995 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
996 rtc::Event* destruction_event) const {
997 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
998 codec_height_);
999 }
1000
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001001 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001002 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001003 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001004
1005 video_source_.IncomingCapturedFrame(
1006 CreateFrame(1, codec_width_, codec_height_));
1007 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +02001008 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001009 }
1010
sprang4847ae62017-06-27 07:06:52 -07001011 void WaitForEncodedFrame(int64_t expected_ntp_time) {
1012 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001013 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001014 }
1015
Markus Handell2cfc1af2022-08-19 08:16:48 +00001016 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, TimeDelta timeout) {
1017 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001018 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001019 return ok;
1020 }
1021
1022 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
1023 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001024 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001025 }
1026
1027 void ExpectDroppedFrame() {
1028 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001029 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001030 }
1031
Markus Handell2cfc1af2022-08-19 08:16:48 +00001032 bool WaitForFrame(TimeDelta timeout) {
1033 bool ok = sink_.WaitForFrame(timeout);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001034 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001035 return ok;
1036 }
1037
perkj26091b12016-09-01 01:17:40 -07001038 class TestEncoder : public test::FakeEncoder {
1039 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001040 explicit TestEncoder(TimeController* time_controller)
1041 : FakeEncoder(time_controller->GetClock()),
1042 time_controller_(time_controller) {
1043 RTC_DCHECK(time_controller_);
1044 }
perkj26091b12016-09-01 01:17:40 -07001045
Erik Språngaed30702018-11-05 12:57:17 +01001046 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +02001047 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +02001048 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +01001049 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001050 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +01001051 info.scaling_settings = VideoEncoder::ScalingSettings(
1052 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001053 }
1054 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001055 for (int i = 0; i < kMaxSpatialLayers; ++i) {
1056 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001057 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001058 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001059 for (int tid = 0; tid < num_layers; ++tid)
1060 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001061 }
1062 }
Erik Språngaed30702018-11-05 12:57:17 +01001063 }
Sergey Silkin6456e352019-07-08 17:56:40 +02001064
1065 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001066 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001067 info.apply_alignment_to_all_simulcast_layers =
1068 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001069 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001070 if (is_qp_trusted_.has_value()) {
1071 info.is_qp_trusted = is_qp_trusted_;
1072 }
Erik Språngaed30702018-11-05 12:57:17 +01001073 return info;
kthelgason876222f2016-11-29 01:44:11 -08001074 }
1075
Erik Språngb7cb7b52019-02-26 15:52:33 +01001076 int32_t RegisterEncodeCompleteCallback(
1077 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +02001078 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001079 encoded_image_callback_ = callback;
1080 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
1081 }
1082
perkjfa10b552016-10-02 23:45:26 -07001083 void ContinueEncode() { continue_encode_event_.Set(); }
1084
1085 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
1086 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +02001087 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -07001088 EXPECT_EQ(timestamp_, timestamp);
1089 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
1090 }
1091
kthelgason2fc52542017-03-03 00:24:41 -08001092 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +02001093 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -08001094 quality_scaling_ = b;
1095 }
kthelgasonad9010c2017-02-14 00:46:51 -08001096
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001097 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +02001098 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001099 requested_resolution_alignment_ = requested_resolution_alignment;
1100 }
1101
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001102 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
1103 MutexLock lock(&local_mutex_);
1104 apply_alignment_to_all_simulcast_layers_ = b;
1105 }
1106
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001107 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +02001108 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001109 is_hardware_accelerated_ = is_hardware_accelerated;
1110 }
1111
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001112 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
1113 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +02001114 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001115 temporal_layers_supported_[spatial_idx] = supported;
1116 }
1117
Sergey Silkin6456e352019-07-08 17:56:40 +02001118 void SetResolutionBitrateLimits(
1119 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +02001120 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +02001121 resolution_bitrate_limits_ = thresholds;
1122 }
1123
sprangfe627f32017-03-29 08:24:59 -07001124 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +02001125 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -07001126 force_init_encode_failed_ = force_failure;
1127 }
1128
Niels Möller6bb5ab92019-01-11 11:11:10 +01001129 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +02001130 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001131 rate_factor_ = rate_factor;
1132 }
1133
Erik Språngd7329ca2019-02-21 21:19:53 +01001134 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +02001135 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001136 return last_framerate_;
1137 }
1138
Erik Språngd7329ca2019-02-21 21:19:53 +01001139 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +02001140 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001141 return last_update_rect_;
1142 }
1143
Niels Möller87e2d782019-03-07 10:18:23 +01001144 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +02001145 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001146 return last_frame_types_;
1147 }
1148
1149 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +01001150 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +01001151 keyframe ? VideoFrameType::kVideoFrameKey
1152 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01001153 {
Markus Handella3765182020-07-08 13:13:32 +02001154 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001155 last_frame_types_ = frame_type;
1156 }
Niels Möllerb859b322019-03-07 12:40:01 +01001157 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +01001158 }
1159
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001160 void InjectEncodedImage(const EncodedImage& image,
1161 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +02001162 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001163 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001164 }
1165
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001166 void SetEncodedImageData(
1167 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +02001168 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001169 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001170 }
1171
Erik Språngd7329ca2019-02-21 21:19:53 +01001172 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +02001173 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001174 expect_null_frame_ = true;
1175 }
1176
Erik Språng5056af02019-09-02 15:53:11 +02001177 absl::optional<VideoEncoder::RateControlParameters>
1178 GetAndResetLastRateControlSettings() {
1179 auto settings = last_rate_control_settings_;
1180 last_rate_control_settings_.reset();
1181 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +01001182 }
1183
Henrik Boström56db9ff2021-03-24 09:06:45 +01001184 int GetLastInputWidth() const {
1185 MutexLock lock(&local_mutex_);
1186 return last_input_width_;
1187 }
1188
1189 int GetLastInputHeight() const {
1190 MutexLock lock(&local_mutex_);
1191 return last_input_height_;
1192 }
1193
Evan Shrubsole895556e2020-10-05 09:15:13 +02001194 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1195 MutexLock lock(&local_mutex_);
1196 return last_input_pixel_format_;
1197 }
1198
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001199 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001200 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001201 return num_set_rates_;
1202 }
1203
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001204 void SetPreferredPixelFormats(
1205 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1206 pixel_formats) {
1207 MutexLock lock(&local_mutex_);
1208 preferred_pixel_formats_ = std::move(pixel_formats);
1209 }
1210
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001211 void SetIsQpTrusted(absl::optional<bool> trusted) {
1212 MutexLock lock(&local_mutex_);
1213 is_qp_trusted_ = trusted;
1214 }
1215
Erik Språnge4589cb2022-04-06 16:44:30 +02001216 VideoCodecComplexity LastEncoderComplexity() {
1217 MutexLock lock(&local_mutex_);
1218 return last_encoder_complexity_;
1219 }
1220
perkjfa10b552016-10-02 23:45:26 -07001221 private:
perkj26091b12016-09-01 01:17:40 -07001222 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001223 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001224 {
Markus Handella3765182020-07-08 13:13:32 +02001225 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001226 if (expect_null_frame_) {
1227 EXPECT_EQ(input_image.timestamp(), 0u);
1228 EXPECT_EQ(input_image.width(), 1);
1229 last_frame_types_ = *frame_types;
1230 expect_null_frame_ = false;
1231 } else {
1232 EXPECT_GT(input_image.timestamp(), timestamp_);
1233 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1234 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1235 }
perkj26091b12016-09-01 01:17:40 -07001236
1237 timestamp_ = input_image.timestamp();
1238 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001239 last_input_width_ = input_image.width();
1240 last_input_height_ = input_image.height();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001241 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001242 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001243 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001244 }
Niels Möllerb859b322019-03-07 12:40:01 +01001245 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001246 return result;
1247 }
1248
Niels Möller08ae7ce2020-09-23 15:58:12 +02001249 CodecSpecificInfo EncodeHook(
1250 EncodedImage& encoded_image,
1251 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001252 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001253 {
1254 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001255 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001256 }
1257 MutexLock lock(&local_mutex_);
1258 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001259 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001260 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001261 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001262 }
1263
sprangfe627f32017-03-29 08:24:59 -07001264 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001265 const Settings& settings) override {
1266 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001267
Markus Handella3765182020-07-08 13:13:32 +02001268 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001269 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001270
Erik Språng82fad3d2018-03-21 09:57:23 +01001271 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001272 // Simulate setting up temporal layers, in order to validate the life
1273 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001274 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001275 frame_buffer_controller_ =
1276 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001277 }
Erik Språnge4589cb2022-04-06 16:44:30 +02001278
1279 last_encoder_complexity_ = config->GetVideoEncoderComplexity();
1280
Erik Språngb7cb7b52019-02-26 15:52:33 +01001281 if (force_init_encode_failed_) {
1282 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001283 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001284 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001285
Erik Språngb7cb7b52019-02-26 15:52:33 +01001286 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001287 return res;
1288 }
1289
Erik Språngb7cb7b52019-02-26 15:52:33 +01001290 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001291 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001292 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1293 initialized_ = EncoderState::kUninitialized;
1294 return FakeEncoder::Release();
1295 }
1296
Erik Språng16cb8f52019-04-12 13:59:09 +02001297 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001298 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001299 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001300 VideoBitrateAllocation adjusted_rate_allocation;
1301 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1302 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001303 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001304 adjusted_rate_allocation.SetBitrate(
1305 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001306 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001307 rate_factor_));
1308 }
1309 }
1310 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001311 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001312 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001313 RateControlParameters adjusted_paramters = parameters;
1314 adjusted_paramters.bitrate = adjusted_rate_allocation;
1315 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001316 }
1317
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001318 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001319 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001320 enum class EncoderState {
1321 kUninitialized,
1322 kInitializationFailed,
1323 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001324 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
perkj26091b12016-09-01 01:17:40 -07001325 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001326 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1327 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1328 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1329 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1330 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1331 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001332 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1333 false;
Markus Handella3765182020-07-08 13:13:32 +02001334 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001335 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1336 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001337 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001338 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001339 absl::optional<bool>
1340 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001341 local_mutex_);
1342 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1343 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1344 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001345 absl::optional<VideoEncoder::RateControlParameters>
1346 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001347 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1348 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001349 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001350 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001351 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1352 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001353 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 17:56:40 +02001354 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001355 RTC_GUARDED_BY(local_mutex_);
1356 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001357 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1358 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001359 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1360 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001361 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
Erik Språnge4589cb2022-04-06 16:44:30 +02001362 VideoCodecComplexity last_encoder_complexity_ RTC_GUARDED_BY(local_mutex_){
1363 VideoCodecComplexity::kComplexityNormal};
perkj26091b12016-09-01 01:17:40 -07001364 };
1365
mflodmancc3d4422017-08-03 08:27:51 -07001366 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001367 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001368 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1369 : time_controller_(time_controller), test_encoder_(test_encoder) {
1370 RTC_DCHECK(time_controller_);
1371 }
perkj26091b12016-09-01 01:17:40 -07001372
perkj26091b12016-09-01 01:17:40 -07001373 void WaitForEncodedFrame(int64_t expected_ntp_time) {
Markus Handell2cfc1af2022-08-19 08:16:48 +00001374 EXPECT_TRUE(TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeout));
sprang4847ae62017-06-27 07:06:52 -07001375 }
1376
1377 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
Markus Handell2cfc1af2022-08-19 08:16:48 +00001378 TimeDelta timeout) {
perkj26091b12016-09-01 01:17:40 -07001379 uint32_t timestamp = 0;
Markus Handell2cfc1af2022-08-19 08:16:48 +00001380 if (!WaitForFrame(timeout))
sprang4847ae62017-06-27 07:06:52 -07001381 return false;
perkj26091b12016-09-01 01:17:40 -07001382 {
Markus Handella3765182020-07-08 13:13:32 +02001383 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001384 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001385 }
1386 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001387 return true;
perkj26091b12016-09-01 01:17:40 -07001388 }
1389
sprangb1ca0732017-02-01 08:38:12 -08001390 void WaitForEncodedFrame(uint32_t expected_width,
1391 uint32_t expected_height) {
Markus Handell2cfc1af2022-08-19 08:16:48 +00001392 EXPECT_TRUE(WaitForFrame(kDefaultTimeout));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001393 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001394 }
1395
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001396 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001397 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001398 uint32_t width = 0;
1399 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001400 {
Markus Handella3765182020-07-08 13:13:32 +02001401 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001402 width = last_width_;
1403 height = last_height_;
1404 }
1405 EXPECT_EQ(expected_height, height);
1406 EXPECT_EQ(expected_width, width);
1407 }
1408
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001409 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1410 VideoRotation rotation;
1411 {
Markus Handella3765182020-07-08 13:13:32 +02001412 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001413 rotation = last_rotation_;
1414 }
1415 EXPECT_EQ(expected_rotation, rotation);
1416 }
1417
Markus Handell2cfc1af2022-08-19 08:16:48 +00001418 void ExpectDroppedFrame() {
1419 EXPECT_FALSE(WaitForFrame(TimeDelta::Millis(100)));
1420 }
kthelgason2bc68642017-02-07 07:02:22 -08001421
Markus Handell2cfc1af2022-08-19 08:16:48 +00001422 bool WaitForFrame(TimeDelta timeout) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001423 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +01001424 time_controller_->AdvanceTime(TimeDelta::Zero());
Markus Handell2cfc1af2022-08-19 08:16:48 +00001425 bool ret = encoded_frame_event_.Wait(timeout);
Markus Handell28c71802021-11-08 10:11:55 +01001426 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001427 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001428 }
1429
perkj26091b12016-09-01 01:17:40 -07001430 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001431 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001432 expect_frames_ = false;
1433 }
1434
asaperssonfab67072017-04-04 05:51:49 -07001435 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001436 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001437 return number_of_reconfigurations_;
1438 }
1439
asaperssonfab67072017-04-04 05:51:49 -07001440 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001441 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001442 return min_transmit_bitrate_bps_;
1443 }
1444
Erik Språngd7329ca2019-02-21 21:19:53 +01001445 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001446 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001447 num_expected_layers_ = num_layers;
1448 }
1449
Erik Språngb7cb7b52019-02-26 15:52:33 +01001450 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001451 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001452 return last_capture_time_ms_;
1453 }
1454
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001455 const EncodedImage& GetLastEncodedImage() {
1456 MutexLock lock(&mutex_);
1457 return last_encoded_image_;
1458 }
1459
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001460 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001461 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001462 return std::move(last_encoded_image_data_);
1463 }
1464
Per Kjellanderdcef6412020-10-07 15:09:05 +02001465 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1466 MutexLock lock(&mutex_);
1467 return last_bitrate_allocation_;
1468 }
1469
1470 int number_of_bitrate_allocations() const {
1471 MutexLock lock(&mutex_);
1472 return number_of_bitrate_allocations_;
1473 }
1474
Per Kjellandera9434842020-10-15 17:53:22 +02001475 VideoLayersAllocation GetLastVideoLayersAllocation() {
1476 MutexLock lock(&mutex_);
1477 return last_layers_allocation_;
1478 }
1479
1480 int number_of_layers_allocations() const {
1481 MutexLock lock(&mutex_);
1482 return number_of_layers_allocations_;
1483 }
1484
perkj26091b12016-09-01 01:17:40 -07001485 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001486 Result OnEncodedImage(
1487 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001488 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001489 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001490 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001491 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001492 last_encoded_image_data_ = std::vector<uint8_t>(
1493 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001494 uint32_t timestamp = encoded_image.Timestamp();
1495 if (last_timestamp_ != timestamp) {
1496 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001497 last_width_ = encoded_image._encodedWidth;
1498 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001499 } else {
1500 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001501 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1502 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001503 }
1504 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001505 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001506 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001507 if (num_received_layers_ == num_expected_layers_) {
1508 encoded_frame_event_.Set();
1509 }
sprangb1ca0732017-02-01 08:38:12 -08001510 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001511 }
1512
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001513 void OnEncoderConfigurationChanged(
1514 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001515 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001516 VideoEncoderConfig::ContentType content_type,
1517 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001518 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001519 ++number_of_reconfigurations_;
1520 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1521 }
1522
Per Kjellanderdcef6412020-10-07 15:09:05 +02001523 void OnBitrateAllocationUpdated(
1524 const VideoBitrateAllocation& allocation) override {
1525 MutexLock lock(&mutex_);
1526 ++number_of_bitrate_allocations_;
1527 last_bitrate_allocation_ = allocation;
1528 }
1529
Per Kjellandera9434842020-10-15 17:53:22 +02001530 void OnVideoLayersAllocationUpdated(
1531 VideoLayersAllocation allocation) override {
1532 MutexLock lock(&mutex_);
1533 ++number_of_layers_allocations_;
1534 last_layers_allocation_ = allocation;
1535 rtc::StringBuilder log;
1536 for (const auto& layer : allocation.active_spatial_layers) {
1537 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1538 << "[";
1539 for (const auto target_bitrate :
1540 layer.target_bitrate_per_temporal_layer) {
1541 log << target_bitrate.kbps() << ",";
1542 }
1543 log << "]";
1544 }
Harald Alvestrand97597c02021-11-04 12:01:23 +00001545 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 17:53:22 +02001546 }
1547
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001548 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001549 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001550 TestEncoder* test_encoder_;
1551 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001552 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001553 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001554 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001555 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001556 uint32_t last_height_ = 0;
1557 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001558 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001559 size_t num_expected_layers_ = 1;
1560 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001561 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001562 int number_of_reconfigurations_ = 0;
1563 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001564 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1565 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001566 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1567 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001568 };
1569
Sergey Silkin5ee69672019-07-02 14:18:34 +02001570 class VideoBitrateAllocatorProxyFactory
1571 : public VideoBitrateAllocatorFactory {
1572 public:
1573 VideoBitrateAllocatorProxyFactory()
1574 : bitrate_allocator_factory_(
1575 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1576
1577 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1578 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001579 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001580 codec_config_ = codec;
1581 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1582 }
1583
1584 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001585 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001586 return codec_config_;
1587 }
1588
1589 private:
1590 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1591
Markus Handella3765182020-07-08 13:13:32 +02001592 mutable Mutex mutex_;
1593 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001594 };
1595
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001596 Clock* clock() { return time_controller_.GetClock(); }
1597 void AdvanceTime(TimeDelta duration) {
1598 time_controller_.AdvanceTime(duration);
1599 }
1600
1601 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1602
1603 protected:
1604 virtual TaskQueueFactory* GetTaskQueueFactory() {
1605 return time_controller_.GetTaskQueueFactory();
1606 }
1607
Jonas Orelandc7f691a2022-03-09 15:12:07 +01001608 test::ScopedKeyValueConfig field_trials_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001609 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001610 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001611 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001612 int codec_width_;
1613 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001614 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001615 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001616 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001617 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001618 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001619 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001620 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001621 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001622};
1623
mflodmancc3d4422017-08-03 08:27:51 -07001624TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001625 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001626 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001627 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001628 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001629 WaitForEncodedFrame(1);
Markus Handell2cfc1af2022-08-19 08:16:48 +00001630 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
mflodmancc3d4422017-08-03 08:27:51 -07001631 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001632}
1633
mflodmancc3d4422017-08-03 08:27:51 -07001634TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001635 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001636 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001637 // The encoder will cache up to one frame for a short duration. Adding two
1638 // frames means that the first frame will be dropped and the second frame will
1639 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001640 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001641 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001642 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Markus Handell28c71802021-11-08 10:11:55 +01001643 AdvanceTime(TimeDelta::Zero());
Markus Handell2cfc1af2022-08-19 08:16:48 +00001644 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
perkj26091b12016-09-01 01:17:40 -07001645
Henrik Boström381d1092020-05-12 18:49:07 +02001646 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001647 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001648
Sebastian Janssona3177052018-04-10 13:05:49 +02001649 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001650 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001651 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1652
1653 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001654 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001655}
1656
mflodmancc3d4422017-08-03 08:27:51 -07001657TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001658 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001659 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001660 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001661 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001662
Henrik Boström381d1092020-05-12 18:49:07 +02001663 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001664 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1665
Sebastian Janssona3177052018-04-10 13:05:49 +02001666 // The encoder will cache up to one frame for a short duration. Adding two
1667 // frames means that the first frame will be dropped and the second frame will
1668 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001669 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001670 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001671
Henrik Boström381d1092020-05-12 18:49:07 +02001672 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001673 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001674 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001675 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1676 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001677 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001678}
1679
mflodmancc3d4422017-08-03 08:27:51 -07001680TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001681 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001682 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001683 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001684 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001685
1686 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001687 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001688
perkja49cbd32016-09-16 07:53:41 -07001689 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001690 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001691 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001692}
1693
mflodmancc3d4422017-08-03 08:27:51 -07001694TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001695 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001696 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001697
perkja49cbd32016-09-16 07:53:41 -07001698 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001699 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001700
mflodmancc3d4422017-08-03 08:27:51 -07001701 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001702 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001703 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001704 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
Markus Handell2cfc1af2022-08-19 08:16:48 +00001705 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
perkj26091b12016-09-01 01:17:40 -07001706}
1707
Markus Handell9a478b52021-11-18 16:07:01 +01001708TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
1709 test::FrameForwarder source;
1710 video_stream_encoder_->SetSource(&source,
1711 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02001712 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001713 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001714
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001715 int dropped_count = 0;
1716 stats_proxy_->SetDroppedFrameCallback(
1717 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1718 ++dropped_count;
1719 });
1720
Markus Handell9a478b52021-11-18 16:07:01 +01001721 source.IncomingCapturedFrame(CreateFrame(1, nullptr));
1722 source.IncomingCapturedFrame(CreateFrame(2, nullptr));
1723 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001724 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001725 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001726}
1727
Henrik Boström56db9ff2021-03-24 09:06:45 +01001728TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001729 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001730 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001731
1732 rtc::Event frame_destroyed_event;
1733 video_source_.IncomingCapturedFrame(
1734 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001735 WaitForEncodedFrame(1);
1736 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1737 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001738 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1739 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001740 video_stream_encoder_->Stop();
1741}
1742
Henrik Boström56db9ff2021-03-24 09:06:45 +01001743TEST_F(VideoStreamEncoderTest,
1744 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001745 // Use the cropping factory.
1746 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001747 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001748 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1749 kMaxPayloadLength);
1750 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1751
1752 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001753 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001754 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001755 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1756 WaitForEncodedFrame(1);
1757 // The encoder will have been configured once.
1758 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001759 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1760 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 06:59:12 -07001761
1762 // Now send in a fake frame that needs to be cropped as the width/height
1763 // aren't divisible by 4 (see CreateEncoderStreams above).
1764 rtc::Event frame_destroyed_event;
1765 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1766 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001767 WaitForEncodedFrame(2);
1768 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1769 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001770 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1771 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001772 video_stream_encoder_->Stop();
1773}
1774
Evan Shrubsole895556e2020-10-05 09:15:13 +02001775TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1776 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001777 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001778
1779 video_source_.IncomingCapturedFrame(
1780 CreateNV12Frame(1, codec_width_, codec_height_));
1781 WaitForEncodedFrame(1);
1782 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1783 fake_encoder_.GetLastInputPixelFormat());
1784 video_stream_encoder_->Stop();
1785}
1786
Henrik Boström56db9ff2021-03-24 09:06:45 +01001787TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001788 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001789 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001790
1791 fake_encoder_.SetPreferredPixelFormats({});
1792
1793 rtc::Event frame_destroyed_event;
1794 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1795 1, &frame_destroyed_event, codec_width_, codec_height_));
1796 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001797 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001798 fake_encoder_.GetLastInputPixelFormat());
1799 video_stream_encoder_->Stop();
1800}
1801
Henrik Boström56db9ff2021-03-24 09:06:45 +01001802TEST_F(VideoStreamEncoderTest,
1803 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001804 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001805 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001806
1807 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1808
1809 rtc::Event frame_destroyed_event;
1810 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1811 1, &frame_destroyed_event, codec_width_, codec_height_));
1812 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001813 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001814 fake_encoder_.GetLastInputPixelFormat());
1815 video_stream_encoder_->Stop();
1816}
1817
Henrik Boström56db9ff2021-03-24 09:06:45 +01001818TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001819 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001820 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001821
1822 // Fake NV12 native frame does not allow mapping to I444.
1823 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1824
1825 rtc::Event frame_destroyed_event;
1826 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1827 1, &frame_destroyed_event, codec_width_, codec_height_));
1828 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001829 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001830 fake_encoder_.GetLastInputPixelFormat());
1831 video_stream_encoder_->Stop();
1832}
1833
Henrik Boström56db9ff2021-03-24 09:06:45 +01001834TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001835 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001836 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001837
1838 rtc::Event frame_destroyed_event;
1839 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1840 1, &frame_destroyed_event, codec_width_, codec_height_));
1841 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001842 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001843 fake_encoder_.GetLastInputPixelFormat());
1844 video_stream_encoder_->Stop();
1845}
1846
Ying Wang9b881ab2020-02-07 14:29:32 +01001847TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001848 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001849 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001850 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1851 WaitForEncodedFrame(1);
1852
Henrik Boström381d1092020-05-12 18:49:07 +02001853 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001854 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001855 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1856 // frames. Adding two frames means that the first frame will be dropped and
1857 // the second frame will be sent to the encoder.
1858 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1859 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1860 WaitForEncodedFrame(3);
1861 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1862 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1863 WaitForEncodedFrame(5);
1864 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1865 video_stream_encoder_->Stop();
1866}
1867
mflodmancc3d4422017-08-03 08:27:51 -07001868TEST_F(VideoStreamEncoderTest,
1869 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001870 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001871 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001872 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001873
1874 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001875 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001876 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001877 // The encoder will have been configured once when the first frame is
1878 // received.
1879 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001880
1881 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001882 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001883 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001884 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001885 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001886
1887 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001888 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001889 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001890 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001891 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001892
mflodmancc3d4422017-08-03 08:27:51 -07001893 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001894}
1895
mflodmancc3d4422017-08-03 08:27:51 -07001896TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001897 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001898 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001899
1900 // Capture a frame and wait for it to synchronize with the encoder thread.
1901 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001902 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001903 // The encoder will have been configured once.
1904 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001905 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1906 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-02 23:45:26 -07001907
1908 codec_width_ *= 2;
1909 codec_height_ *= 2;
1910 // Capture a frame with a higher resolution and wait for it to synchronize
1911 // with the encoder thread.
1912 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001913 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 10:07:11 +02001914 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1915 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 21:37:57 +01001916 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001917
mflodmancc3d4422017-08-03 08:27:51 -07001918 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001919}
1920
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001921TEST_F(VideoStreamEncoderTest,
1922 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001923 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001924 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001925
1926 // Capture a frame and wait for it to synchronize with the encoder thread.
1927 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1928 WaitForEncodedFrame(1);
1929
1930 VideoEncoderConfig video_encoder_config;
1931 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1932 // Changing the max payload data length recreates encoder.
1933 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1934 kMaxPayloadLength / 2);
1935
1936 // Capture a frame and wait for it to synchronize with the encoder thread.
1937 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1938 WaitForEncodedFrame(2);
1939 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1940
1941 video_stream_encoder_->Stop();
1942}
1943
Sergey Silkin5ee69672019-07-02 14:18:34 +02001944TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001945 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001946 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001947
1948 VideoEncoderConfig video_encoder_config;
1949 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02001950 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1951 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001952 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1953 kMaxPayloadLength);
1954
1955 // Capture a frame and wait for it to synchronize with the encoder thread.
1956 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1957 WaitForEncodedFrame(1);
1958 // The encoder will have been configured once when the first frame is
1959 // received.
1960 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001961 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001962 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001963 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001964 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1965
Sergey Silkin6456e352019-07-08 17:56:40 +02001966 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1967 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 10:07:11 +02001968 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1969 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001970 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1971 kMaxPayloadLength);
1972
1973 // Capture a frame and wait for it to synchronize with the encoder thread.
1974 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1975 WaitForEncodedFrame(2);
1976 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1977 // Bitrate limits have changed - rate allocator should be reconfigured,
1978 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 10:07:11 +02001979 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001980 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001981 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001982 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001983 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001984
1985 video_stream_encoder_->Stop();
1986}
1987
Sergey Silkin6456e352019-07-08 17:56:40 +02001988TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001989 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001990 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001991 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001992
Sergey Silkincd02eba2020-01-20 14:48:40 +01001993 const uint32_t kMinEncBitrateKbps = 100;
1994 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001995 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001996 /*frame_size_pixels=*/codec_width_ * codec_height_,
1997 /*min_start_bitrate_bps=*/0,
1998 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1999 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002000 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2001
Sergey Silkincd02eba2020-01-20 14:48:40 +01002002 VideoEncoderConfig video_encoder_config;
2003 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2004 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
2005 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2006 (kMinEncBitrateKbps + 1) * 1000;
2007 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2008 kMaxPayloadLength);
2009
2010 // When both encoder and app provide bitrate limits, the intersection of
2011 // provided sets should be used.
2012 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2013 WaitForEncodedFrame(1);
2014 EXPECT_EQ(kMaxEncBitrateKbps,
2015 bitrate_allocator_factory_.codec_config().maxBitrate);
2016 EXPECT_EQ(kMinEncBitrateKbps + 1,
2017 bitrate_allocator_factory_.codec_config().minBitrate);
2018
2019 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
2020 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2021 (kMinEncBitrateKbps - 1) * 1000;
2022 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2023 kMaxPayloadLength);
2024 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002025 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002026 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002027 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002028 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002029 bitrate_allocator_factory_.codec_config().minBitrate);
2030
Sergey Silkincd02eba2020-01-20 14:48:40 +01002031 video_stream_encoder_->Stop();
2032}
2033
2034TEST_F(VideoStreamEncoderTest,
2035 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02002036 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002037 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002038
2039 const uint32_t kMinAppBitrateKbps = 100;
2040 const uint32_t kMaxAppBitrateKbps = 200;
2041 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
2042 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
2043 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2044 /*frame_size_pixels=*/codec_width_ * codec_height_,
2045 /*min_start_bitrate_bps=*/0,
2046 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
2047 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
2048 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2049
2050 VideoEncoderConfig video_encoder_config;
2051 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2052 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
2053 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2054 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002055 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2056 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002057
Sergey Silkincd02eba2020-01-20 14:48:40 +01002058 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2059 WaitForEncodedFrame(1);
2060 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002061 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002062 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002063 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02002064
2065 video_stream_encoder_->Stop();
2066}
2067
2068TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002069 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02002070 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002071 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02002072
2073 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002074 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002075 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002076 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002077 fake_encoder_.SetResolutionBitrateLimits(
2078 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
2079
2080 VideoEncoderConfig video_encoder_config;
2081 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2082 video_encoder_config.max_bitrate_bps = 0;
2083 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2084 kMaxPayloadLength);
2085
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002086 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002087 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2088 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002089 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2090 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002091 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2092 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2093
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002094 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002095 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2096 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002097 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2098 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002099 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2100 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2101
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002102 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02002103 // encoder for 360p should be used.
2104 video_source_.IncomingCapturedFrame(
2105 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2106 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002107 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2108 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002109 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2110 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2111
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002112 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02002113 // ignored.
2114 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2115 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002116 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2117 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002118 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2119 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002120 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2121 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002122 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2123 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2124
2125 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2126 // for 270p should be used.
2127 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2128 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002129 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2130 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002131 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2132 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2133
2134 video_stream_encoder_->Stop();
2135}
2136
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002137TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002138 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002139 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002140
2141 VideoEncoderConfig video_encoder_config;
2142 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2143 video_encoder_config.max_bitrate_bps = 0;
2144 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2145 kMaxPayloadLength);
2146
2147 // Encode 720p frame to get the default encoder target bitrate.
2148 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2149 WaitForEncodedFrame(1);
2150 const uint32_t kDefaultTargetBitrateFor720pKbps =
2151 bitrate_allocator_factory_.codec_config()
2152 .simulcastStream[0]
2153 .targetBitrate;
2154
2155 // Set the max recommended encoder bitrate to something lower than the default
2156 // target bitrate.
2157 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2158 1280 * 720, 10 * 1000, 10 * 1000,
2159 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2160 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2161
2162 // Change resolution to trigger encoder reinitialization.
2163 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2164 WaitForEncodedFrame(2);
2165 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2166 WaitForEncodedFrame(3);
2167
2168 // Ensure the target bitrate is capped by the max bitrate.
2169 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2170 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2171 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2172 .simulcastStream[0]
2173 .targetBitrate *
2174 1000,
2175 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2176
2177 video_stream_encoder_->Stop();
2178}
2179
Åsa Perssona7e34d32021-01-20 15:36:13 +01002180TEST_F(VideoStreamEncoderTest,
2181 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2182 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2183 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2184 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2185 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2186 fake_encoder_.SetResolutionBitrateLimits(
2187 {kEncoderLimits270p, kEncoderLimits360p});
2188
2189 // Two streams, highest stream active.
2190 VideoEncoderConfig config;
2191 const int kNumStreams = 2;
2192 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2193 config.max_bitrate_bps = 0;
2194 config.simulcast_layers[0].active = false;
2195 config.simulcast_layers[1].active = true;
2196 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002197 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002198 "VP8", /*max qp*/ 56, /*screencast*/ false,
2199 /*screenshare enabled*/ false);
2200 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2201
2202 // The encoder bitrate limits for 270p should be used.
2203 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
Erik Språngf449af82022-08-08 17:54:55 +02002204 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002205 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002206 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002207 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002208 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002209 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002210
2211 // The encoder bitrate limits for 360p should be used.
2212 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
Erik Språngf449af82022-08-08 17:54:55 +02002213 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002214 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002215 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002216 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002217 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002218
2219 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2220 video_source_.IncomingCapturedFrame(
2221 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
Erik Språngf449af82022-08-08 17:54:55 +02002222 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002223 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002224 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002225 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002226 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002227
2228 // Resolution higher than 360p. Encoder limits should be ignored.
2229 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02002230 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002231 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002232 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002233 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002234 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002235 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002236 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002237 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002238 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002239
2240 // Resolution lower than 270p. The encoder limits for 270p should be used.
2241 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
Erik Språngf449af82022-08-08 17:54:55 +02002242 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002243 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002244 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002245 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002246 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002247
2248 video_stream_encoder_->Stop();
2249}
2250
2251TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002252 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2253 // Two streams, highest stream active.
2254 VideoEncoderConfig config;
2255 const int kNumStreams = 2;
2256 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2257 config.max_bitrate_bps = 0;
2258 config.simulcast_layers[0].active = false;
2259 config.simulcast_layers[1].active = true;
2260 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002261 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002262 "VP8", /*max qp*/ 56, /*screencast*/ false,
2263 /*screenshare enabled*/ false);
2264 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2265
2266 // Default bitrate limits for 270p should be used.
2267 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2268 kDefaultLimits270p =
2269 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002270 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002271 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
Erik Språngf449af82022-08-08 17:54:55 +02002272 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002273 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 10:39:51 +01002274 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002275 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002276 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002277 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002278
2279 // Default bitrate limits for 360p should be used.
2280 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2281 kDefaultLimits360p =
2282 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002283 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002284 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
Erik Språngf449af82022-08-08 17:54:55 +02002285 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Persson258e9892021-02-25 10:39:51 +01002286 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002287 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002288 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002289 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002290
2291 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2292 video_source_.IncomingCapturedFrame(
2293 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
Erik Språngf449af82022-08-08 17:54:55 +02002294 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Persson258e9892021-02-25 10:39:51 +01002295 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002296 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002297 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002298 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002299
2300 // Default bitrate limits for 540p should be used.
2301 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2302 kDefaultLimits540p =
2303 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002304 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002305 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02002306 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Persson258e9892021-02-25 10:39:51 +01002307 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002308 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002309 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002310 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002311
2312 video_stream_encoder_->Stop();
2313}
2314
2315TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002316 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2317 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2318 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2319 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2320 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2321 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2322 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2323 fake_encoder_.SetResolutionBitrateLimits(
2324 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2325
2326 // Three streams, middle stream active.
2327 VideoEncoderConfig config;
2328 const int kNumStreams = 3;
2329 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2330 config.simulcast_layers[0].active = false;
2331 config.simulcast_layers[1].active = true;
2332 config.simulcast_layers[2].active = false;
2333 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002334 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002335 "VP8", /*max qp*/ 56, /*screencast*/ false,
2336 /*screenshare enabled*/ false);
2337 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2338
2339 // The encoder bitrate limits for 360p should be used.
2340 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02002341 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002342 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002343 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002344 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002345 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002346 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002347
2348 // The encoder bitrate limits for 270p should be used.
2349 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02002350 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002351 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002352 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002353 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002354 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002355
2356 video_stream_encoder_->Stop();
2357}
2358
2359TEST_F(VideoStreamEncoderTest,
2360 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2361 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2362 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2363 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2364 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2365 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2366 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2367 fake_encoder_.SetResolutionBitrateLimits(
2368 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2369
2370 // Three streams, lowest stream active.
2371 VideoEncoderConfig config;
2372 const int kNumStreams = 3;
2373 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2374 config.simulcast_layers[0].active = true;
2375 config.simulcast_layers[1].active = false;
2376 config.simulcast_layers[2].active = false;
2377 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002378 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002379 "VP8", /*max qp*/ 56, /*screencast*/ false,
2380 /*screenshare enabled*/ false);
2381 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2382
2383 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2384 // on lowest stream, limits for 270p should not be used
2385 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02002386 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002387 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002388 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002389 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002390 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002391 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002392
2393 video_stream_encoder_->Stop();
2394}
2395
2396TEST_F(VideoStreamEncoderTest,
2397 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2398 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2399 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2400 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2401 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2402 fake_encoder_.SetResolutionBitrateLimits(
2403 {kEncoderLimits270p, kEncoderLimits360p});
2404 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2405
2406 // Two streams, highest stream active.
2407 VideoEncoderConfig config;
2408 const int kNumStreams = 2;
2409 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2410 config.simulcast_layers[0].active = false;
2411 config.simulcast_layers[1].active = true;
2412 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2413 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002414 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002415 "VP8", /*max qp*/ 56, /*screencast*/ false,
2416 /*screenshare enabled*/ false);
2417 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2418
2419 // The encoder bitrate limits for 270p should be used.
2420 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
Erik Språngf449af82022-08-08 17:54:55 +02002421 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002422 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002423 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002424 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002425 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002426 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002427
2428 // The max configured bitrate is less than the encoder limit for 360p.
2429 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
Erik Språngf449af82022-08-08 17:54:55 +02002430 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002431 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002432 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002433 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002434 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002435
2436 video_stream_encoder_->Stop();
2437}
2438
mflodmancc3d4422017-08-03 08:27:51 -07002439TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002440 EXPECT_TRUE(video_source_.has_sinks());
2441 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002442 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002443 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002444 EXPECT_FALSE(video_source_.has_sinks());
2445 EXPECT_TRUE(new_video_source.has_sinks());
2446
mflodmancc3d4422017-08-03 08:27:51 -07002447 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002448}
2449
mflodmancc3d4422017-08-03 08:27:51 -07002450TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002451 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002452 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002453 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002454 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002455}
2456
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002457class ResolutionAlignmentTest
2458 : public VideoStreamEncoderTest,
2459 public ::testing::WithParamInterface<
2460 ::testing::tuple<int, std::vector<double>>> {
2461 public:
2462 ResolutionAlignmentTest()
2463 : requested_alignment_(::testing::get<0>(GetParam())),
2464 scale_factors_(::testing::get<1>(GetParam())) {}
2465
2466 protected:
2467 const int requested_alignment_;
2468 const std::vector<double> scale_factors_;
2469};
2470
2471INSTANTIATE_TEST_SUITE_P(
2472 AlignmentAndScaleFactors,
2473 ResolutionAlignmentTest,
2474 ::testing::Combine(
2475 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2476 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2477 std::vector<double>{-1.0, -1.0},
2478 std::vector<double>{-1.0, -1.0, -1.0},
2479 std::vector<double>{4.0, 2.0, 1.0},
2480 std::vector<double>{9999.0, -1.0, 1.0},
2481 std::vector<double>{3.99, 2.01, 1.0},
2482 std::vector<double>{4.9, 1.7, 1.25},
2483 std::vector<double>{10.0, 4.0, 3.0},
2484 std::vector<double>{1.75, 3.5},
2485 std::vector<double>{1.5, 2.5},
2486 std::vector<double>{1.3, 1.0})));
2487
2488TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2489 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002490 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002491 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2492 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2493
2494 // Fill config with the scaling factor by which to reduce encoding size.
2495 const int num_streams = scale_factors_.size();
2496 VideoEncoderConfig config;
2497 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2498 for (int i = 0; i < num_streams; ++i) {
2499 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2500 }
2501 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002502 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002503 "VP8", /*max qp*/ 56, /*screencast*/ false,
2504 /*screenshare enabled*/ false);
2505 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2506
Henrik Boström381d1092020-05-12 18:49:07 +02002507 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002508 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2509 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002510 // Wait for all layers before triggering event.
2511 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002512
2513 // On the 1st frame, we should have initialized the encoder and
2514 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002515 int64_t timestamp_ms = kFrameIntervalMs;
2516 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2517 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002518 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002519
2520 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2521 // (It's up the to the encoder to potentially drop the previous frame,
2522 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002523 timestamp_ms += kFrameIntervalMs;
2524 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2525 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002526 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002527
Asa Persson606d3cb2021-10-04 10:07:11 +02002528 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002529 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2530 // Frame size should be a multiple of the requested alignment.
2531 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2532 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2533 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2534 // Aspect ratio should match.
2535 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2536 codec.height * codec.simulcastStream[i].width);
2537 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002538
2539 video_stream_encoder_->Stop();
2540}
2541
Jonathan Yubc771b72017-12-08 17:04:29 -08002542TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2543 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002544 const int kWidth = 1280;
2545 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002546
2547 // We rely on the automatic resolution adaptation, but we handle framerate
2548 // adaptation manually by mocking the stats proxy.
2549 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002550
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002551 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002552 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002553 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002554 video_stream_encoder_->SetSource(&video_source_,
2555 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002556 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002557 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002558 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002559 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2560
Jonathan Yubc771b72017-12-08 17:04:29 -08002561 // Adapt down as far as possible.
2562 rtc::VideoSinkWants last_wants;
2563 int64_t t = 1;
2564 int loop_count = 0;
2565 do {
2566 ++loop_count;
2567 last_wants = video_source_.sink_wants();
2568
2569 // Simulate the framerate we've been asked to adapt to.
2570 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2571 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2572 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2573 mock_stats.input_frame_rate = fps;
2574 stats_proxy_->SetMockStats(mock_stats);
2575
2576 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2577 sink_.WaitForEncodedFrame(t);
2578 t += frame_interval_ms;
2579
mflodmancc3d4422017-08-03 08:27:51 -07002580 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002581 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002582 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002583 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2584 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002585 } while (video_source_.sink_wants().max_pixel_count <
2586 last_wants.max_pixel_count ||
2587 video_source_.sink_wants().max_framerate_fps <
2588 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002589
Jonathan Yubc771b72017-12-08 17:04:29 -08002590 // Verify that we've adapted all the way down.
2591 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002592 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002593 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2594 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002595 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002596 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2597 *video_source_.last_sent_height());
2598 EXPECT_EQ(kMinBalancedFramerateFps,
2599 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002600
Jonathan Yubc771b72017-12-08 17:04:29 -08002601 // Adapt back up the same number of times we adapted down.
2602 for (int i = 0; i < loop_count - 1; ++i) {
2603 last_wants = video_source_.sink_wants();
2604
2605 // Simulate the framerate we've been asked to adapt to.
2606 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2607 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2608 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2609 mock_stats.input_frame_rate = fps;
2610 stats_proxy_->SetMockStats(mock_stats);
2611
2612 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2613 sink_.WaitForEncodedFrame(t);
2614 t += frame_interval_ms;
2615
Henrik Boström91aa7322020-04-28 12:24:33 +02002616 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002617 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002618 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002619 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2620 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002621 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2622 last_wants.max_pixel_count ||
2623 video_source_.sink_wants().max_framerate_fps >
2624 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002625 }
2626
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002627 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002628 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002629 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002630 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2631 EXPECT_EQ((loop_count - 1) * 2,
2632 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002633
mflodmancc3d4422017-08-03 08:27:51 -07002634 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002635}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002636
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002637TEST_F(VideoStreamEncoderTest,
2638 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02002639 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2640 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002641 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002642
2643 const int kFrameWidth = 1280;
2644 const int kFrameHeight = 720;
2645
2646 int64_t ntp_time = kFrameIntervalMs;
2647
2648 // Force an input frame rate to be available, or the adaptation call won't
2649 // know what framerate to adapt form.
2650 const int kInputFps = 30;
2651 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2652 stats.input_frame_rate = kInputFps;
2653 stats_proxy_->SetMockStats(stats);
2654
2655 video_source_.set_adaptation_enabled(true);
2656 video_stream_encoder_->SetSource(
2657 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002658 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002659 video_source_.IncomingCapturedFrame(
2660 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2661 sink_.WaitForEncodedFrame(ntp_time);
2662 ntp_time += kFrameIntervalMs;
2663
2664 // Trigger CPU overuse.
2665 video_stream_encoder_->TriggerCpuOveruse();
2666 video_source_.IncomingCapturedFrame(
2667 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2668 sink_.WaitForEncodedFrame(ntp_time);
2669 ntp_time += kFrameIntervalMs;
2670
2671 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2672 EXPECT_EQ(std::numeric_limits<int>::max(),
2673 video_source_.sink_wants().max_pixel_count);
2674 // Some framerate constraint should be set.
2675 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2676 EXPECT_LT(restricted_fps, kInputFps);
2677 video_source_.IncomingCapturedFrame(
2678 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2679 sink_.WaitForEncodedFrame(ntp_time);
2680 ntp_time += 100;
2681
Henrik Boström2671dac2020-05-19 16:29:09 +02002682 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002683 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2684 // Give the encoder queue time to process the change in degradation preference
2685 // by waiting for an encoded frame.
2686 video_source_.IncomingCapturedFrame(
2687 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2688 sink_.WaitForEncodedFrame(ntp_time);
2689 ntp_time += kFrameIntervalMs;
2690
2691 video_stream_encoder_->TriggerQualityLow();
2692 video_source_.IncomingCapturedFrame(
2693 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2694 sink_.WaitForEncodedFrame(ntp_time);
2695 ntp_time += kFrameIntervalMs;
2696
2697 // Some resolution constraint should be set.
2698 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2699 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2700 kFrameWidth * kFrameHeight);
2701 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2702
2703 int pixel_count = video_source_.sink_wants().max_pixel_count;
2704 // Triggering a CPU underuse should not change the sink wants since it has
2705 // not been overused for resolution since we changed degradation preference.
2706 video_stream_encoder_->TriggerCpuUnderuse();
2707 video_source_.IncomingCapturedFrame(
2708 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2709 sink_.WaitForEncodedFrame(ntp_time);
2710 ntp_time += kFrameIntervalMs;
2711 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2712 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2713
Evan Shrubsole64469032020-06-11 10:45:29 +02002714 // Change the degradation preference back. CPU underuse should not adapt since
2715 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002716 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002717 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2718 video_source_.IncomingCapturedFrame(
2719 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2720 sink_.WaitForEncodedFrame(ntp_time);
2721 ntp_time += 100;
2722 // Resolution adaptations is gone after changing degradation preference.
2723 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2724 EXPECT_EQ(std::numeric_limits<int>::max(),
2725 video_source_.sink_wants().max_pixel_count);
2726 // The fps adaptation from above is now back.
2727 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2728
2729 // Trigger CPU underuse.
2730 video_stream_encoder_->TriggerCpuUnderuse();
2731 video_source_.IncomingCapturedFrame(
2732 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2733 sink_.WaitForEncodedFrame(ntp_time);
2734 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002735 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2736
2737 // Trigger QP underuse, fps should return to normal.
2738 video_stream_encoder_->TriggerQualityHigh();
2739 video_source_.IncomingCapturedFrame(
2740 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2741 sink_.WaitForEncodedFrame(ntp_time);
2742 ntp_time += kFrameIntervalMs;
2743 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002744
2745 video_stream_encoder_->Stop();
2746}
2747
mflodmancc3d4422017-08-03 08:27:51 -07002748TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002749 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002750 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002751 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002752
sprangc5d62e22017-04-02 23:53:04 -07002753 const int kFrameWidth = 1280;
2754 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002755
Åsa Persson8c1bf952018-09-13 10:42:19 +02002756 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002757
kthelgason5e13d412016-12-01 03:59:51 -08002758 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002759 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002760 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002761 frame_timestamp += kFrameIntervalMs;
2762
perkj803d97f2016-11-01 11:45:46 -07002763 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002764 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002765 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002766 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002767 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002768 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002769
asapersson0944a802017-04-07 00:57:58 -07002770 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002771 // wanted resolution.
2772 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2773 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2774 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002775 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002776
2777 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002778 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002779 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002780 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002781 // Give the encoder queue time to process the change in degradation preference
2782 // by waiting for an encoded frame.
2783 new_video_source.IncomingCapturedFrame(
2784 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2785 sink_.WaitForEncodedFrame(frame_timestamp);
2786 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002787 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002788 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002789
sprangc5d62e22017-04-02 23:53:04 -07002790 // Force an input frame rate to be available, or the adaptation call won't
2791 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002792 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002793 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002794 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002795 stats_proxy_->SetMockStats(stats);
2796
mflodmancc3d4422017-08-03 08:27:51 -07002797 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002798 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002799 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002800 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002801 frame_timestamp += kFrameIntervalMs;
2802
2803 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002804 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002805 EXPECT_EQ(std::numeric_limits<int>::max(),
2806 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002807 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002808
asapersson02465b82017-04-10 01:12:52 -07002809 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002810 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2811 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002812 // Give the encoder queue time to process the change in degradation preference
2813 // by waiting for an encoded frame.
2814 new_video_source.IncomingCapturedFrame(
2815 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2816 sink_.WaitForEncodedFrame(frame_timestamp);
2817 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002818 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002819
mflodmancc3d4422017-08-03 08:27:51 -07002820 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002821 new_video_source.IncomingCapturedFrame(
2822 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002823 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002824 frame_timestamp += kFrameIntervalMs;
2825
2826 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002827 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002828
2829 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002830 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002831 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002832 // Give the encoder queue time to process the change in degradation preference
2833 // by waiting for an encoded frame.
2834 new_video_source.IncomingCapturedFrame(
2835 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2836 sink_.WaitForEncodedFrame(frame_timestamp);
2837 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002838 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2839 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002840 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002841 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002842
2843 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002844 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002845 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002846 // Give the encoder queue time to process the change in degradation preference
2847 // by waiting for an encoded frame.
2848 new_video_source.IncomingCapturedFrame(
2849 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2850 sink_.WaitForEncodedFrame(frame_timestamp);
2851 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002852 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2853 EXPECT_EQ(std::numeric_limits<int>::max(),
2854 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002855 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002856
mflodmancc3d4422017-08-03 08:27:51 -07002857 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002858}
2859
mflodmancc3d4422017-08-03 08:27:51 -07002860TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002861 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002862 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002863
asaperssonfab67072017-04-04 05:51:49 -07002864 const int kWidth = 1280;
2865 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002866 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002867 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002868 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2869 EXPECT_FALSE(stats.bw_limited_resolution);
2870 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2871
2872 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002873 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002874 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002875 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002876
2877 stats = stats_proxy_->GetStats();
2878 EXPECT_TRUE(stats.bw_limited_resolution);
2879 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2880
2881 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002882 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002883 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002884 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002885
2886 stats = stats_proxy_->GetStats();
2887 EXPECT_FALSE(stats.bw_limited_resolution);
2888 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2889 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2890
mflodmancc3d4422017-08-03 08:27:51 -07002891 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002892}
2893
mflodmancc3d4422017-08-03 08:27:51 -07002894TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002895 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002896 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002897
2898 const int kWidth = 1280;
2899 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002900 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002901 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002902 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2903 EXPECT_FALSE(stats.cpu_limited_resolution);
2904 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2905
2906 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002907 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002908 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002909 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002910
2911 stats = stats_proxy_->GetStats();
2912 EXPECT_TRUE(stats.cpu_limited_resolution);
2913 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2914
2915 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002916 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002917 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002918 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002919
2920 stats = stats_proxy_->GetStats();
2921 EXPECT_FALSE(stats.cpu_limited_resolution);
2922 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002923 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002924
mflodmancc3d4422017-08-03 08:27:51 -07002925 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002926}
2927
mflodmancc3d4422017-08-03 08:27:51 -07002928TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002929 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002930 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002931
asaperssonfab67072017-04-04 05:51:49 -07002932 const int kWidth = 1280;
2933 const int kHeight = 720;
2934 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002935 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002936 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002937 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002938 EXPECT_FALSE(stats.cpu_limited_resolution);
2939 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2940
asaperssonfab67072017-04-04 05:51:49 -07002941 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002942 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002943 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002944 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002945 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002946 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002947 EXPECT_TRUE(stats.cpu_limited_resolution);
2948 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2949
2950 // Set new source with adaptation still enabled.
2951 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002952 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002953 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002954
asaperssonfab67072017-04-04 05:51:49 -07002955 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002956 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002957 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002958 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002959 EXPECT_TRUE(stats.cpu_limited_resolution);
2960 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2961
2962 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002963 video_stream_encoder_->SetSource(&new_video_source,
2964 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002965
asaperssonfab67072017-04-04 05:51:49 -07002966 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002967 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002968 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002969 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002970 EXPECT_FALSE(stats.cpu_limited_resolution);
2971 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2972
2973 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002974 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002975 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002976
asaperssonfab67072017-04-04 05:51:49 -07002977 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002978 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002979 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002980 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002981 EXPECT_TRUE(stats.cpu_limited_resolution);
2982 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2983
asaperssonfab67072017-04-04 05:51:49 -07002984 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002985 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002986 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002987 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002988 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002989 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002990 EXPECT_FALSE(stats.cpu_limited_resolution);
2991 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002992 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002993
mflodmancc3d4422017-08-03 08:27:51 -07002994 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002995}
2996
mflodmancc3d4422017-08-03 08:27:51 -07002997TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002998 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002999 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003000
asaperssonfab67072017-04-04 05:51:49 -07003001 const int kWidth = 1280;
3002 const int kHeight = 720;
3003 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003004 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003005 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003006 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003007 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003008 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003009
3010 // Set new source with adaptation still enabled.
3011 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003012 video_stream_encoder_->SetSource(&new_video_source,
3013 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08003014
asaperssonfab67072017-04-04 05:51:49 -07003015 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003016 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08003017 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003018 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003019 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003020 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003021
asaperssonfab67072017-04-04 05:51:49 -07003022 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003023 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003024 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003025 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003026 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003027 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003028 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003029 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003030
asaperssonfab67072017-04-04 05:51:49 -07003031 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003032 video_stream_encoder_->SetSource(&new_video_source,
3033 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08003034
asaperssonfab67072017-04-04 05:51:49 -07003035 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003036 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003037 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003038 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003039 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003040 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003041
asapersson02465b82017-04-10 01:12:52 -07003042 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07003043 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003044 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003045
asaperssonfab67072017-04-04 05:51:49 -07003046 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003047 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08003048 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003049 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003050 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003051 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
3052 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003053
mflodmancc3d4422017-08-03 08:27:51 -07003054 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003055}
3056
mflodmancc3d4422017-08-03 08:27:51 -07003057TEST_F(VideoStreamEncoderTest,
3058 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02003059 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003060 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07003061
3062 const int kWidth = 1280;
3063 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02003064 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07003065 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003066 video_source_.IncomingCapturedFrame(
3067 CreateFrame(timestamp_ms, kWidth, kHeight));
3068 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003069 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3070 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3071 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3072
3073 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003074 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003075 timestamp_ms += kFrameIntervalMs;
3076 video_source_.IncomingCapturedFrame(
3077 CreateFrame(timestamp_ms, kWidth, kHeight));
3078 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003079 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3080 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3081 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3082
3083 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07003084 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003085 timestamp_ms += kFrameIntervalMs;
3086 video_source_.IncomingCapturedFrame(
3087 CreateFrame(timestamp_ms, kWidth, kHeight));
3088 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003089 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3090 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3091 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3092
Niels Möller4db138e2018-04-19 09:04:13 +02003093 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07003094 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003095
3096 VideoEncoderConfig video_encoder_config;
3097 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3098 // Make format different, to force recreation of encoder.
3099 video_encoder_config.video_format.parameters["foo"] = "foo";
3100 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003101 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003102 timestamp_ms += kFrameIntervalMs;
3103 video_source_.IncomingCapturedFrame(
3104 CreateFrame(timestamp_ms, kWidth, kHeight));
3105 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003106 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3107 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3108 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3109
mflodmancc3d4422017-08-03 08:27:51 -07003110 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003111}
3112
mflodmancc3d4422017-08-03 08:27:51 -07003113TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003114 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003115 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003116 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003117
3118 const int kWidth = 1280;
3119 const int kHeight = 720;
3120 int sequence = 1;
3121
3122 // Enable BALANCED preference, no initial limitation.
3123 test::FrameForwarder source;
3124 video_stream_encoder_->SetSource(&source,
3125 webrtc::DegradationPreference::BALANCED);
3126 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3127 WaitForEncodedFrame(sequence++);
3128 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3129 EXPECT_FALSE(stats.cpu_limited_resolution);
3130 EXPECT_FALSE(stats.cpu_limited_framerate);
3131 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3132
3133 // Trigger CPU overuse, should now adapt down.
3134 video_stream_encoder_->TriggerCpuOveruse();
3135 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3136 WaitForEncodedFrame(sequence++);
3137 stats = stats_proxy_->GetStats();
3138 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3139
3140 // Set new degradation preference should clear restrictions since we changed
3141 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003142 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003143 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3144 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3145 WaitForEncodedFrame(sequence++);
3146 stats = stats_proxy_->GetStats();
3147 EXPECT_FALSE(stats.cpu_limited_resolution);
3148 EXPECT_FALSE(stats.cpu_limited_framerate);
3149 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3150
3151 // Force an input frame rate to be available, or the adaptation call won't
3152 // know what framerate to adapt from.
3153 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3154 mock_stats.input_frame_rate = 30;
3155 stats_proxy_->SetMockStats(mock_stats);
3156 video_stream_encoder_->TriggerCpuOveruse();
3157 stats_proxy_->ResetMockStats();
3158 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3159 WaitForEncodedFrame(sequence++);
3160
3161 // We have now adapted once.
3162 stats = stats_proxy_->GetStats();
3163 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3164
3165 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003166 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3167 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003168 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3169 WaitForEncodedFrame(sequence++);
3170 stats = stats_proxy_->GetStats();
3171 EXPECT_FALSE(stats.cpu_limited_resolution);
3172 EXPECT_FALSE(stats.cpu_limited_framerate);
3173 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3174
3175 video_stream_encoder_->Stop();
3176}
3177
3178TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003179 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003180 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003181 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003182
asapersson0944a802017-04-07 00:57:58 -07003183 const int kWidth = 1280;
3184 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003185 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003186
asaperssonfab67072017-04-04 05:51:49 -07003187 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003188 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003189 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003190 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003191 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003192 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3193
asapersson02465b82017-04-10 01:12:52 -07003194 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003195 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003196 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003197 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003198 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003199 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003200 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003201 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3202
3203 // Set new source with adaptation still enabled.
3204 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003205 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003206 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003207
3208 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003209 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003210 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003211 stats = stats_proxy_->GetStats();
3212 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003213 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003214 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3215
sprangc5d62e22017-04-02 23:53:04 -07003216 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003217 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003218 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003219 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003220 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003221 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003222 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003223 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003224 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003225 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003226 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3227
sprangc5d62e22017-04-02 23:53:04 -07003228 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003229 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003230 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3231 mock_stats.input_frame_rate = 30;
3232 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003233 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003234 stats_proxy_->ResetMockStats();
3235
3236 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003237 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003238 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003239
3240 // Framerate now adapted.
3241 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003242 EXPECT_FALSE(stats.cpu_limited_resolution);
3243 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003244 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3245
3246 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003247 video_stream_encoder_->SetSource(&new_video_source,
3248 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003249 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003250 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003251 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003252
3253 stats = stats_proxy_->GetStats();
3254 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003255 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003256 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3257
3258 // Try to trigger overuse. Should not succeed.
3259 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003260 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003261 stats_proxy_->ResetMockStats();
3262
3263 stats = stats_proxy_->GetStats();
3264 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003265 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003266 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3267
3268 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003269 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003270 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003271 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003272 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003273 stats = stats_proxy_->GetStats();
3274 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003275 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003276 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003277
3278 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003279 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003280 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003281 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003282 stats = stats_proxy_->GetStats();
3283 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003284 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003285 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3286
3287 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003288 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003289 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003290 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003291 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003292 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003293 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003294 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003295 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003296 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003297 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3298
3299 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003300 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003301 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003302 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003303 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003304 stats = stats_proxy_->GetStats();
3305 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003306 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003307 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003308 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003309
mflodmancc3d4422017-08-03 08:27:51 -07003310 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003311}
3312
mflodmancc3d4422017-08-03 08:27:51 -07003313TEST_F(VideoStreamEncoderTest,
3314 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003315 const int kWidth = 1280;
3316 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003317 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003318 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003319
asaperssonfab67072017-04-04 05:51:49 -07003320 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003321 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003322
asaperssonfab67072017-04-04 05:51:49 -07003323 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003324 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003325
asaperssonfab67072017-04-04 05:51:49 -07003326 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003327 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003328
asaperssonfab67072017-04-04 05:51:49 -07003329 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003330 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003331
kthelgason876222f2016-11-29 01:44:11 -08003332 // Expect a scale down.
3333 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003334 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003335
asapersson02465b82017-04-10 01:12:52 -07003336 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003337 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003338 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003339 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003340
asaperssonfab67072017-04-04 05:51:49 -07003341 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003342 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003343 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003344 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003345
asaperssonfab67072017-04-04 05:51:49 -07003346 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003347 EXPECT_EQ(std::numeric_limits<int>::max(),
3348 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003349
asaperssonfab67072017-04-04 05:51:49 -07003350 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003351 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003352 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003353 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003354
asapersson02465b82017-04-10 01:12:52 -07003355 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003356 EXPECT_EQ(std::numeric_limits<int>::max(),
3357 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003358
mflodmancc3d4422017-08-03 08:27:51 -07003359 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003360}
3361
mflodmancc3d4422017-08-03 08:27:51 -07003362TEST_F(VideoStreamEncoderTest,
3363 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003364 const int kWidth = 1280;
3365 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003366 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003367 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003368
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003369 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003370 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003371 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003372 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003373
3374 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003375 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003376 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003377 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3378 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3379
3380 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003381 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003382 EXPECT_THAT(source.sink_wants(),
3383 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003384 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3385 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3386 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3387
3388 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003389 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003390 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3391 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3392 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3393
mflodmancc3d4422017-08-03 08:27:51 -07003394 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003395}
3396
mflodmancc3d4422017-08-03 08:27:51 -07003397TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003398 const int kWidth = 1280;
3399 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003400 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003401 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003402
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003403 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003404 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003405 video_stream_encoder_->SetSource(&source,
3406 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003407 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3408 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003409 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003410
3411 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003412 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003413 EXPECT_THAT(source.sink_wants(),
3414 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003415 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3416 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3417 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3418
3419 // Trigger adapt down for same input resolution, expect no change.
3420 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3421 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003422 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003423 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3424 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3425 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3426
3427 // Trigger adapt down for larger input resolution, expect no change.
3428 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3429 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003430 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003431 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3432 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3433 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3434
mflodmancc3d4422017-08-03 08:27:51 -07003435 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003436}
3437
mflodmancc3d4422017-08-03 08:27:51 -07003438TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003439 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3440 const int kWidth = 640;
3441 const int kHeight = 360;
3442 const int64_t kFrameIntervalMs = 150;
3443 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003444 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003445
3446 // Enable BALANCED preference, no initial limitation.
3447 AdaptingFrameForwarder source(&time_controller_);
3448 source.set_adaptation_enabled(true);
3449 video_stream_encoder_->SetSource(&source,
3450 webrtc::DegradationPreference::BALANCED);
3451
3452 int64_t timestamp_ms = kFrameIntervalMs;
3453 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3454 sink_.WaitForEncodedFrame(kWidth, kHeight);
3455 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3456 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3457 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3458 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3459
3460 // Trigger adapt down, expect reduced fps (640x360@15fps).
3461 video_stream_encoder_->TriggerQualityLow();
3462 timestamp_ms += kFrameIntervalMs;
3463 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3464 sink_.WaitForEncodedFrame(timestamp_ms);
3465 EXPECT_THAT(source.sink_wants(),
3466 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3467 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3468 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3469 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3470
3471 // Source requests 270p, expect reduced resolution (480x270@15fps).
3472 source.OnOutputFormatRequest(480, 270);
3473 timestamp_ms += kFrameIntervalMs;
3474 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3475 WaitForEncodedFrame(480, 270);
3476 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3477
3478 // Trigger adapt down, expect reduced fps (480x270@10fps).
3479 video_stream_encoder_->TriggerQualityLow();
3480 timestamp_ms += kFrameIntervalMs;
3481 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3482 sink_.WaitForEncodedFrame(timestamp_ms);
3483 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3484 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3485 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3486 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3487
3488 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3489 source.OnOutputFormatRequest(320, 180);
3490 timestamp_ms += kFrameIntervalMs;
3491 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3492 WaitForEncodedFrame(320, 180);
3493 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3494
3495 // Trigger adapt down, expect reduced fps (320x180@7fps).
3496 video_stream_encoder_->TriggerQualityLow();
3497 timestamp_ms += kFrameIntervalMs;
3498 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3499 sink_.WaitForEncodedFrame(timestamp_ms);
3500 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3501 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3502 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3503 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3504
3505 // Source requests VGA, expect increased resolution (640x360@7fps).
3506 source.OnOutputFormatRequest(640, 360);
3507 timestamp_ms += kFrameIntervalMs;
3508 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3509 WaitForEncodedFrame(timestamp_ms);
3510 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3511
3512 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3513 video_stream_encoder_->TriggerQualityHigh();
3514 timestamp_ms += kFrameIntervalMs;
3515 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3516 WaitForEncodedFrame(timestamp_ms);
3517 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3518 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3519 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3520 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3521
3522 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3523 video_stream_encoder_->TriggerQualityHigh();
3524 timestamp_ms += kFrameIntervalMs;
3525 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3526 WaitForEncodedFrame(timestamp_ms);
3527 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3528 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3529 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3530 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3531
3532 // Trigger adapt up, expect increased fps (640x360@maxfps).
3533 video_stream_encoder_->TriggerQualityHigh();
3534 timestamp_ms += kFrameIntervalMs;
3535 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3536 WaitForEncodedFrame(timestamp_ms);
3537 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3538 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3539 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3540 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3541
3542 video_stream_encoder_->Stop();
3543}
3544
3545TEST_F(VideoStreamEncoderTest,
3546 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3547 const int kWidth = 1280;
3548 const int kHeight = 720;
3549 const int64_t kFrameIntervalMs = 150;
3550 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003551 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003552
3553 // Enable BALANCED preference, no initial limitation.
3554 AdaptingFrameForwarder source(&time_controller_);
3555 source.set_adaptation_enabled(true);
3556 video_stream_encoder_->SetSource(&source,
3557 webrtc::DegradationPreference::BALANCED);
3558
3559 int64_t timestamp_ms = kFrameIntervalMs;
3560 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3561 sink_.WaitForEncodedFrame(kWidth, kHeight);
3562 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3563 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3564 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3565 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3566
3567 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3568 video_stream_encoder_->TriggerQualityLow();
3569 timestamp_ms += kFrameIntervalMs;
3570 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3571 sink_.WaitForEncodedFrame(timestamp_ms);
3572 EXPECT_THAT(source.sink_wants(),
3573 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3574 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3575 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3576 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3577
3578 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3579 video_stream_encoder_->TriggerQualityLow();
3580 timestamp_ms += kFrameIntervalMs;
3581 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3582 sink_.WaitForEncodedFrame(timestamp_ms);
3583 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3584 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3585 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3586 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3587
3588 // Trigger adapt down, expect reduced fps (640x360@15fps).
3589 video_stream_encoder_->TriggerQualityLow();
3590 timestamp_ms += kFrameIntervalMs;
3591 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3592 WaitForEncodedFrame(timestamp_ms);
3593 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3594 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3595 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3596 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3597
3598 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3599 source.OnOutputFormatRequest(320, 180);
3600 timestamp_ms += kFrameIntervalMs;
3601 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3602 WaitForEncodedFrame(320, 180);
3603 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3604 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3605
3606 // Trigger adapt down, expect reduced fps (320x180@7fps).
3607 video_stream_encoder_->TriggerCpuOveruse();
3608 timestamp_ms += kFrameIntervalMs;
3609 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3610 WaitForEncodedFrame(timestamp_ms);
3611 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3612 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3613 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3614 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3615 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3616 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3617 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3618
3619 // Source requests HD, expect increased resolution (640x360@7fps).
3620 source.OnOutputFormatRequest(1280, 720);
3621 timestamp_ms += kFrameIntervalMs;
3622 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3623 WaitForEncodedFrame(timestamp_ms);
3624 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3625 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3626
3627 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3628 video_stream_encoder_->TriggerCpuUnderuse();
3629 timestamp_ms += kFrameIntervalMs;
3630 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3631 WaitForEncodedFrame(timestamp_ms);
3632 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3633 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3634 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3635 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3636 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3637 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3638 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3639
3640 // Trigger adapt up, expect increased fps (640x360@maxfps).
3641 video_stream_encoder_->TriggerQualityHigh();
3642 video_stream_encoder_->TriggerCpuUnderuse();
3643 timestamp_ms += kFrameIntervalMs;
3644 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3645 WaitForEncodedFrame(timestamp_ms);
3646 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3647 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3648 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3649 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3650 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3651 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3652 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3653
3654 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3655 video_stream_encoder_->TriggerQualityHigh();
3656 video_stream_encoder_->TriggerCpuUnderuse();
3657 timestamp_ms += kFrameIntervalMs;
3658 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3659 WaitForEncodedFrame(timestamp_ms);
3660 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3661 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3662 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3663 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3664 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3665 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3666 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3667
3668 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3669 video_stream_encoder_->TriggerQualityHigh();
3670 video_stream_encoder_->TriggerCpuUnderuse();
3671 timestamp_ms += kFrameIntervalMs;
3672 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3673 WaitForEncodedFrame(timestamp_ms);
3674 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3675 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3676 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3677 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3678 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3679 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3680 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3681
3682 video_stream_encoder_->Stop();
3683}
3684
3685TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003686 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003687 const int kWidth = 1280;
3688 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003689 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003690 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003691
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003692 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003693 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003694 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003695 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003696
3697 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003698 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003699 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003700 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3701 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3702
3703 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003704 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003705 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003706 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3707 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3708
mflodmancc3d4422017-08-03 08:27:51 -07003709 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003710}
3711
mflodmancc3d4422017-08-03 08:27:51 -07003712TEST_F(VideoStreamEncoderTest,
3713 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003714 const int kWidth = 1280;
3715 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003716 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003717 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003718
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003719 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003720 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003721 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003722 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003723
3724 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003725 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003726 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003727 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003728 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3729
3730 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003731 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003732 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003733 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003734 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3735
mflodmancc3d4422017-08-03 08:27:51 -07003736 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003737}
3738
mflodmancc3d4422017-08-03 08:27:51 -07003739TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003740 const int kWidth = 1280;
3741 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003742 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003743 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003744
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003745 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003746 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003747 video_stream_encoder_->SetSource(&source,
3748 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003749
3750 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3751 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003752 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003753 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3754 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3755 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3756
3757 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003758 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003759 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003760 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3761 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3762 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3763
mflodmancc3d4422017-08-03 08:27:51 -07003764 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003765}
3766
mflodmancc3d4422017-08-03 08:27:51 -07003767TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003768 const int kWidth = 1280;
3769 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003770 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003771 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003772
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003773 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003774 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003775 video_stream_encoder_->SetSource(&source,
3776 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003777
3778 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3779 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003780 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003781 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3782 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3783 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3784
3785 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003786 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003787 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003788 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3789 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3790 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3791
mflodmancc3d4422017-08-03 08:27:51 -07003792 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003793}
3794
mflodmancc3d4422017-08-03 08:27:51 -07003795TEST_F(VideoStreamEncoderTest,
3796 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003797 const int kWidth = 1280;
3798 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003799 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003800 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003801
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003802 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003803 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003804 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003805 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003806 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003807
3808 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003809 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003810 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003811 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3812 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3813
3814 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003815 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003816 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003817 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003818 EXPECT_THAT(source.sink_wants(),
3819 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003820 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3821 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3822
3823 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003824 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003825 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003826 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3827 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3828 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3829
mflodmancc3d4422017-08-03 08:27:51 -07003830 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003831}
3832
mflodmancc3d4422017-08-03 08:27:51 -07003833TEST_F(VideoStreamEncoderTest,
3834 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003835 const int kWidth = 1280;
3836 const int kHeight = 720;
3837 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003838 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003839 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003840
3841 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3842 stats.input_frame_rate = kInputFps;
3843 stats_proxy_->SetMockStats(stats);
3844
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003845 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003846 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3847 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003848 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003849
3850 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003851 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003852 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3853 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003854 EXPECT_THAT(video_source_.sink_wants(),
3855 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003856
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003857 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003858 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003859 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003860 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003861 // Give the encoder queue time to process the change in degradation preference
3862 // by waiting for an encoded frame.
3863 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3864 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003865 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003866
3867 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003868 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003869 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3870 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003871 EXPECT_THAT(new_video_source.sink_wants(),
3872 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003873
3874 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003875 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003876 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003877
mflodmancc3d4422017-08-03 08:27:51 -07003878 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003879}
3880
mflodmancc3d4422017-08-03 08:27:51 -07003881TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003882 const int kWidth = 1280;
3883 const int kHeight = 720;
3884 const size_t kNumFrames = 10;
3885
Henrik Boström381d1092020-05-12 18:49:07 +02003886 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003887 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003888
asaperssond0de2952017-04-21 01:47:31 -07003889 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003890 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003891 video_source_.set_adaptation_enabled(true);
3892
3893 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3894 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3895
3896 int downscales = 0;
3897 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003898 video_source_.IncomingCapturedFrame(
3899 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3900 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003901
asaperssonfab67072017-04-04 05:51:49 -07003902 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003903 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003904 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003905 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003906
3907 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3908 ++downscales;
3909
3910 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3911 EXPECT_EQ(downscales,
3912 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3913 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003914 }
mflodmancc3d4422017-08-03 08:27:51 -07003915 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003916}
3917
mflodmancc3d4422017-08-03 08:27:51 -07003918TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003919 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3920 const int kWidth = 1280;
3921 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003922 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003923 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003924
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003925 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003926 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003927 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003928 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003929 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003930
Åsa Persson8c1bf952018-09-13 10:42:19 +02003931 int64_t timestamp_ms = kFrameIntervalMs;
3932 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003933 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003934 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003935 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3936 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3937
3938 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003939 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003940 timestamp_ms += kFrameIntervalMs;
3941 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3942 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003943 EXPECT_THAT(source.sink_wants(),
3944 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003945 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3946 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3947
3948 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003949 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003950 timestamp_ms += kFrameIntervalMs;
3951 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003952 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003953 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003954 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3955 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3956
3957 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003958 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003959 timestamp_ms += kFrameIntervalMs;
3960 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3961 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003962 EXPECT_THAT(source.sink_wants(),
3963 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003964 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3965 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3966
3967 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003968 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003969 timestamp_ms += kFrameIntervalMs;
3970 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003971 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003972 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003973 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3974 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3975
mflodmancc3d4422017-08-03 08:27:51 -07003976 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003977}
3978
mflodmancc3d4422017-08-03 08:27:51 -07003979TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003980 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3981 const int kWidth = 1280;
3982 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003983 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003984 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003985
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003986 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003987 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003988 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003989 video_stream_encoder_->SetSource(&source,
3990 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003991
Åsa Persson8c1bf952018-09-13 10:42:19 +02003992 int64_t timestamp_ms = kFrameIntervalMs;
3993 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003994 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003995 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003996 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3997 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3998
3999 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07004000 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004001 timestamp_ms += kFrameIntervalMs;
4002 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4003 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004004 EXPECT_THAT(source.sink_wants(),
4005 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004006 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4007 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4008
4009 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07004010 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004011 timestamp_ms += kFrameIntervalMs;
4012 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07004013 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004014 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004015 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4016 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4017
4018 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07004019 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004020 timestamp_ms += kFrameIntervalMs;
4021 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4022 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004023 EXPECT_THAT(source.sink_wants(),
4024 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004025 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4026 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4027
4028 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07004029 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004030 timestamp_ms += kFrameIntervalMs;
4031 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07004032 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004033 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004034 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4035 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4036
mflodmancc3d4422017-08-03 08:27:51 -07004037 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004038}
4039
Sergey Silkin41c650b2019-10-14 13:12:19 +02004040TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
4041 fake_encoder_.SetResolutionBitrateLimits(
4042 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4043
Henrik Boström381d1092020-05-12 18:49:07 +02004044 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004045 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4046 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4047 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4048 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004049
4050 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004051 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004052 source.set_adaptation_enabled(true);
4053 video_stream_encoder_->SetSource(
4054 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4055
4056 // Insert 720p frame.
4057 int64_t timestamp_ms = kFrameIntervalMs;
4058 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4059 WaitForEncodedFrame(1280, 720);
4060
4061 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02004062 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004063 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4064 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4065 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4066 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004067 video_stream_encoder_->TriggerQualityLow();
4068
4069 // Insert 720p frame. It should be downscaled and encoded.
4070 timestamp_ms += kFrameIntervalMs;
4071 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4072 WaitForEncodedFrame(960, 540);
4073
4074 // Trigger adapt up. Higher resolution should not be requested duo to lack
4075 // of bitrate.
4076 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004077 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02004078
4079 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02004080 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004081 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4082 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4083 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4084 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004085
4086 // Trigger adapt up. Higher resolution should be requested.
4087 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004088 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02004089
4090 video_stream_encoder_->Stop();
4091}
4092
4093TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
4094 fake_encoder_.SetResolutionBitrateLimits(
4095 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4096
4097 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02004098 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004099 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4100 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4101 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4102 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004103
4104 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004105 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004106 source.set_adaptation_enabled(true);
4107 video_stream_encoder_->SetSource(
4108 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4109
4110 // Insert 720p frame. It should be dropped and lower resolution should be
4111 // requested.
4112 int64_t timestamp_ms = kFrameIntervalMs;
4113 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4114 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004115 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004116
4117 // Insert 720p frame. It should be downscaled and encoded.
4118 timestamp_ms += kFrameIntervalMs;
4119 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4120 WaitForEncodedFrame(960, 540);
4121
4122 video_stream_encoder_->Stop();
4123}
4124
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004125class BalancedDegradationTest : public VideoStreamEncoderTest {
4126 protected:
4127 void SetupTest() {
4128 // Reset encoder for field trials to take effect.
4129 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02004130 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004131
4132 // Enable BALANCED preference.
4133 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02004134 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4135 }
4136
Asa Persson606d3cb2021-10-04 10:07:11 +02004137 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02004138 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004139 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004140 }
4141
Åsa Persson45b176f2019-09-30 11:19:05 +02004142 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004143 timestamp_ms_ += kFrameIntervalMs;
4144 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02004145 }
4146
4147 void InsertFrameAndWaitForEncoded() {
4148 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004149 sink_.WaitForEncodedFrame(timestamp_ms_);
4150 }
4151
4152 const int kWidth = 640; // pixels:640x360=230400
4153 const int kHeight = 360;
4154 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4155 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004156 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004157};
4158
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004159TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004160 test::ScopedKeyValueConfig field_trials(
4161 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004162 "WebRTC-Video-BalancedDegradationSettings/"
4163 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4164 SetupTest();
4165
4166 // Force input frame rate.
4167 const int kInputFps = 24;
4168 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4169 stats.input_frame_rate = kInputFps;
4170 stats_proxy_->SetMockStats(stats);
4171
Åsa Persson45b176f2019-09-30 11:19:05 +02004172 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004173 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004174
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004175 // Trigger adapt down, expect scaled down framerate and resolution,
4176 // since Fps diff (input-requested:0) < threshold.
4177 video_stream_encoder_->TriggerQualityLow();
4178 EXPECT_THAT(source_.sink_wants(),
4179 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004180
4181 video_stream_encoder_->Stop();
4182}
4183
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004184TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004185 test::ScopedKeyValueConfig field_trials(
4186 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004187 "WebRTC-Video-BalancedDegradationSettings/"
4188 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4189 SetupTest();
4190
4191 // Force input frame rate.
4192 const int kInputFps = 25;
4193 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4194 stats.input_frame_rate = kInputFps;
4195 stats_proxy_->SetMockStats(stats);
4196
Åsa Persson45b176f2019-09-30 11:19:05 +02004197 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004198 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004199
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004200 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4201 // Fps diff (input-requested:1) == threshold.
4202 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004203 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004204
4205 video_stream_encoder_->Stop();
4206}
4207
4208TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004209 test::ScopedKeyValueConfig field_trials(
4210 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004211 "WebRTC-Video-BalancedDegradationSettings/"
4212 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4213 SetupTest();
4214
4215 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4216
Åsa Persson45b176f2019-09-30 11:19:05 +02004217 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004218 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004219
4220 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4221 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004222 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004223
4224 video_stream_encoder_->Stop();
4225}
4226
Åsa Perssonccfb3402019-09-25 15:13:04 +02004227TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004228 test::ScopedKeyValueConfig field_trials(
4229 field_trials_,
Åsa Persson1b247f12019-08-14 17:26:39 +02004230 "WebRTC-Video-BalancedDegradationSettings/"
4231 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004232 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004233
Asa Persson606d3cb2021-10-04 10:07:11 +02004234 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4235 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4236 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004237
Åsa Persson45b176f2019-09-30 11:19:05 +02004238 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004239 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004240 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4241
4242 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4243 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004244 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004245 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004246 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4247
4248 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4249 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004250 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004251 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004252 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4253
Åsa Persson30ab0152019-08-27 12:22:33 +02004254 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4255 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004256 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004257 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004258 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004259 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4260
4261 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004262 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004263 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004264 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004265
Åsa Persson30ab0152019-08-27 12:22:33 +02004266 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004267 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004268 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004269 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004270 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004271 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4272
4273 video_stream_encoder_->Stop();
4274}
4275
Åsa Perssonccfb3402019-09-25 15:13:04 +02004276TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004277 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004278 test::ScopedKeyValueConfig field_trials(
4279 field_trials_,
Åsa Persson45b176f2019-09-30 11:19:05 +02004280 "WebRTC-Video-BalancedDegradationSettings/"
4281 "pixels:57600|129600|230400,fps:7|24|24/");
4282 SetupTest();
Asa Persson606d3cb2021-10-04 10:07:11 +02004283 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 11:19:05 +02004284
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004285 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004286
4287 // Insert frame, expect scaled down:
4288 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4289 InsertFrame();
Markus Handell2cfc1af2022-08-19 08:16:48 +00004290 EXPECT_FALSE(WaitForFrame(TimeDelta::Seconds(1)));
Åsa Persson45b176f2019-09-30 11:19:05 +02004291 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4292 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4293
4294 // Insert frame, expect scaled down:
4295 // resolution (320x180@24fps).
4296 InsertFrame();
Markus Handell2cfc1af2022-08-19 08:16:48 +00004297 EXPECT_FALSE(WaitForFrame(TimeDelta::Seconds(1)));
Åsa Persson45b176f2019-09-30 11:19:05 +02004298 EXPECT_LT(source_.sink_wants().max_pixel_count,
4299 source_.last_wants().max_pixel_count);
4300 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4301
4302 // Frame should not be dropped (min pixels per frame reached).
4303 InsertFrameAndWaitForEncoded();
4304
4305 video_stream_encoder_->Stop();
4306}
4307
4308TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004309 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004310 test::ScopedKeyValueConfig field_trials(
4311 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004312 "WebRTC-Video-BalancedDegradationSettings/"
4313 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004314 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004315
Asa Persson606d3cb2021-10-04 10:07:11 +02004316 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4317 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4318 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004319
Åsa Persson45b176f2019-09-30 11:19:05 +02004320 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004321 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004322 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4323
4324 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4325 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004326 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004327 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004328 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4329
4330 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4331 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004332 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004333 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004334 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4335
4336 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4337 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004338 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004339 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004340 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4341
Åsa Persson30ab0152019-08-27 12:22:33 +02004342 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4343 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004344 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004345 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004346 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4347
4348 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4349 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004350 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004351 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4352
4353 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004354 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004355 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004356 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004357 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004358 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4359
4360 video_stream_encoder_->Stop();
4361}
4362
Åsa Perssonccfb3402019-09-25 15:13:04 +02004363TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004364 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004365 test::ScopedKeyValueConfig field_trials(
4366 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004367 "WebRTC-Video-BalancedDegradationSettings/"
4368 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004369 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004370
Asa Persson606d3cb2021-10-04 10:07:11 +02004371 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4372 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4373 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4374 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4375 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004376
Åsa Persson45b176f2019-09-30 11:19:05 +02004377 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004378 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004379 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4380
4381 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4382 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004383 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004384 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004385 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4386
4387 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4388 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004389 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004390 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004391 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4392
4393 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4394 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004395 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004396 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004397 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4398
4399 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4400 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004401 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004402 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4403
4404 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004405 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004406 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004407 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004408 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004409 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4410
4411 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004412 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004413 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004414 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004415 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4416
4417 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004418 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004419 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004420 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004421 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004422 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4423
Åsa Persson1b247f12019-08-14 17:26:39 +02004424 video_stream_encoder_->Stop();
4425}
4426
mflodmancc3d4422017-08-03 08:27:51 -07004427TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004428 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4429 const int kWidth = 1280;
4430 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004431 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004432 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004433
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004434 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004435 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004436 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004437 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004438 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004439
Åsa Persson8c1bf952018-09-13 10:42:19 +02004440 int64_t timestamp_ms = kFrameIntervalMs;
4441 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004442 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004443 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004444 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4445 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4446 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4447 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4448
4449 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004450 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004451 timestamp_ms += kFrameIntervalMs;
4452 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4453 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004454 EXPECT_THAT(source.sink_wants(),
4455 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004456 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4457 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4458 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4459 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4460
4461 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004462 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004463 timestamp_ms += kFrameIntervalMs;
4464 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4465 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004466 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004467 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4468 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4469 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4470 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4471
Jonathan Yubc771b72017-12-08 17:04:29 -08004472 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004473 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004474 timestamp_ms += kFrameIntervalMs;
4475 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4476 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004477 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004478 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4479 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004480 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004481 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4482
Jonathan Yubc771b72017-12-08 17:04:29 -08004483 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004484 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004485 timestamp_ms += kFrameIntervalMs;
4486 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4487 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004488 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004489 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004490 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4491 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4492 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4493 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4494
Jonathan Yubc771b72017-12-08 17:04:29 -08004495 // Trigger quality adapt down, expect no change (min resolution reached).
4496 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004497 timestamp_ms += kFrameIntervalMs;
4498 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4499 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004500 EXPECT_THAT(source.sink_wants(), FpsMax());
4501 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004502 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4503 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4504 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4505 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4506
Evan Shrubsole64469032020-06-11 10:45:29 +02004507 // Trigger quality adapt up, expect upscaled resolution (480x270).
4508 video_stream_encoder_->TriggerQualityHigh();
4509 timestamp_ms += kFrameIntervalMs;
4510 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4511 WaitForEncodedFrame(timestamp_ms);
4512 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4513 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4514 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4515 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4516 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4517
4518 // Trigger quality and cpu adapt up since both are most limited, expect
4519 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004520 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004521 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004522 timestamp_ms += kFrameIntervalMs;
4523 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4524 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004525 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004526 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4527 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4528 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004529 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004530
Evan Shrubsole64469032020-06-11 10:45:29 +02004531 // Trigger quality and cpu adapt up since both are most limited, expect
4532 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004533 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004534 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004535 timestamp_ms += kFrameIntervalMs;
4536 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4537 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004538 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004539 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004540 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004541 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004542 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4543 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004544
Evan Shrubsole64469032020-06-11 10:45:29 +02004545 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4546 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004547 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004548 timestamp_ms += kFrameIntervalMs;
4549 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4550 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004551 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004552 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4553 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004554 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004555 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004556
4557 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004558 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004559 timestamp_ms += kFrameIntervalMs;
4560 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004561 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004562 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004563 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004564 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4565 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004566 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004567 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004568
mflodmancc3d4422017-08-03 08:27:51 -07004569 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004570}
4571
mflodmancc3d4422017-08-03 08:27:51 -07004572TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004573 const int kWidth = 640;
4574 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004575
Henrik Boström381d1092020-05-12 18:49:07 +02004576 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004577 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004578
perkj803d97f2016-11-01 11:45:46 -07004579 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004580 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004581 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004582 }
4583
mflodmancc3d4422017-08-03 08:27:51 -07004584 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004585 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004586 video_source_.IncomingCapturedFrame(CreateFrame(
4587 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004588 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004589 }
4590
mflodmancc3d4422017-08-03 08:27:51 -07004591 video_stream_encoder_->Stop();
4592 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004593 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004594
Ying Wangef3998f2019-12-09 13:06:53 +01004595 EXPECT_METRIC_EQ(
4596 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4597 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004598 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4599}
4600
mflodmancc3d4422017-08-03 08:27:51 -07004601TEST_F(VideoStreamEncoderTest,
4602 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004603 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004604 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004605 const int kWidth = 640;
4606 const int kHeight = 360;
4607
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004608 video_stream_encoder_->SetSource(&video_source_,
4609 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004610
4611 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4612 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004613 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004614 }
4615
mflodmancc3d4422017-08-03 08:27:51 -07004616 video_stream_encoder_->Stop();
4617 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004618 stats_proxy_.reset();
4619
4620 EXPECT_EQ(0,
4621 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4622}
4623
Per Kjellanderdcef6412020-10-07 15:09:05 +02004624TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4625 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004626 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004627 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004628
4629 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004630 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 10:07:11 +02004631 SimulcastRateAllocator(fake_encoder_.config())
4632 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004633 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004634
Henrik Boström381d1092020-05-12 18:49:07 +02004635 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004636 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004637
sprang57c2fff2017-01-16 06:24:02 -08004638 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004639 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4640 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004641 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4642 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4643
Erik Språngd7329ca2019-02-21 21:19:53 +01004644 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004645 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004646 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004647
Per Kjellanderdcef6412020-10-07 15:09:05 +02004648 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004649 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004650 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4651 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004652 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004653 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004654
Per Kjellanderdcef6412020-10-07 15:09:05 +02004655 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004656 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004657 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004658 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004659 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4660 WaitForEncodedFrame(CurrentTimeMs());
4661 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004662 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004663 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004664
mflodmancc3d4422017-08-03 08:27:51 -07004665 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004666}
4667
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004668TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004669 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004670 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004671 kVideoLayersAllocation);
4672
4673 const int kDefaultFps = 30;
4674
4675 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004676 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004677
4678 video_source_.IncomingCapturedFrame(
4679 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4680 WaitForEncodedFrame(CurrentTimeMs());
4681 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4682 VideoLayersAllocation last_layer_allocation =
4683 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004684 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004685 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4686
4687 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004688 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004689 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 10:07:11 +02004690 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 17:53:22 +02004691 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4692
Erik Språng9d69cbe2020-10-22 17:44:42 +02004693 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004694 int number_of_layers_allocation = 1;
4695 const int64_t start_time_ms = CurrentTimeMs();
4696 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4697 video_source_.IncomingCapturedFrame(
4698 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4699 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004700 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4701 number_of_layers_allocation = sink_.number_of_layers_allocations();
4702 VideoLayersAllocation new_allocation =
4703 sink_.GetLastVideoLayersAllocation();
4704 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4705 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4706 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4707 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4708 .target_bitrate_per_temporal_layer,
4709 last_layer_allocation.active_spatial_layers[0]
4710 .target_bitrate_per_temporal_layer);
4711 last_layer_allocation = new_allocation;
4712 }
4713 }
4714 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4715 video_stream_encoder_->Stop();
4716}
4717
4718TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004719 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004720 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4721 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4722 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004723 VideoEncoderConfig video_encoder_config;
4724 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4725 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004726 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004727 video_encoder_config.content_type =
4728 VideoEncoderConfig::ContentType::kRealtimeVideo;
4729 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004730 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004731 VideoEncoder::GetDefaultVp8Settings());
4732 for (auto& layer : video_encoder_config.simulcast_layers) {
4733 layer.num_temporal_layers = 2;
4734 }
4735 // Simulcast layers are used for enabling/disabling streams.
4736 video_encoder_config.simulcast_layers[0].active = true;
4737 video_encoder_config.simulcast_layers[1].active = false;
4738 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004739 ConfigureEncoder(std::move(video_encoder_config),
4740 VideoStreamEncoder::BitrateAllocationCallbackType::
4741 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004742
4743 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004744 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004745
4746 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4747 WaitForEncodedFrame(CurrentTimeMs());
4748 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4749 VideoLayersAllocation last_layer_allocation =
4750 sink_.GetLastVideoLayersAllocation();
4751
4752 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4753 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4754 .target_bitrate_per_temporal_layer,
4755 SizeIs(2));
4756 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4757 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4758 video_stream_encoder_->Stop();
4759}
4760
4761TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004762 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004763 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4764 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4765 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004766 VideoEncoderConfig video_encoder_config;
4767 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4768 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004769 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004770 video_encoder_config.content_type =
4771 VideoEncoderConfig::ContentType::kRealtimeVideo;
4772 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004773 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004774 VideoEncoder::GetDefaultVp8Settings());
4775 for (auto& layer : video_encoder_config.simulcast_layers) {
4776 layer.num_temporal_layers = 2;
4777 }
4778 // Simulcast layers are used for enabling/disabling streams.
4779 video_encoder_config.simulcast_layers[0].active = true;
4780 video_encoder_config.simulcast_layers[1].active = false;
4781 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004782 ConfigureEncoder(std::move(video_encoder_config),
4783 VideoStreamEncoder::BitrateAllocationCallbackType::
4784 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004785
4786 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004787 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004788
4789 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4790 WaitForEncodedFrame(CurrentTimeMs());
4791 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4792 VideoLayersAllocation last_layer_allocation =
4793 sink_.GetLastVideoLayersAllocation();
4794
4795 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4796 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4797 .target_bitrate_per_temporal_layer,
4798 SizeIs(2));
4799 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4800
4801 video_stream_encoder_->Stop();
4802}
4803
4804TEST_F(VideoStreamEncoderTest,
4805 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4806 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4807 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004808 VideoEncoderConfig video_encoder_config;
4809 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4810 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004811 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004812 video_encoder_config.content_type =
4813 VideoEncoderConfig::ContentType::kRealtimeVideo;
4814 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4815 vp9_settings.numberOfSpatialLayers = 2;
4816 vp9_settings.numberOfTemporalLayers = 2;
4817 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4818 vp9_settings.automaticResizeOn = false;
4819 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004820 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004821 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004822 ConfigureEncoder(std::move(video_encoder_config),
4823 VideoStreamEncoder::BitrateAllocationCallbackType::
4824 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004825
4826 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004827 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004828
4829 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4830 WaitForEncodedFrame(CurrentTimeMs());
4831 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4832 VideoLayersAllocation last_layer_allocation =
4833 sink_.GetLastVideoLayersAllocation();
4834
4835 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4836 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4837 .target_bitrate_per_temporal_layer,
4838 SizeIs(2));
4839 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4840 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4841 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4842 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4843 .target_bitrate_per_temporal_layer,
4844 SizeIs(2));
4845 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4846 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4847 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4848
4849 // Since full SVC is used, expect the top layer to utilize the full target
4850 // rate.
4851 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4852 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004853 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004854 video_stream_encoder_->Stop();
4855}
4856
4857TEST_F(VideoStreamEncoderTest,
4858 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4859 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4860 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004861 VideoEncoderConfig video_encoder_config;
4862 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4863 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004864 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004865 video_encoder_config.content_type =
4866 VideoEncoderConfig::ContentType::kRealtimeVideo;
4867 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4868 vp9_settings.numberOfSpatialLayers = 2;
4869 vp9_settings.numberOfTemporalLayers = 2;
4870 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4871 vp9_settings.automaticResizeOn = false;
4872 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004873 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004874 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004875 ConfigureEncoder(std::move(video_encoder_config),
4876 VideoStreamEncoder::BitrateAllocationCallbackType::
4877 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004878
4879 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004880 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004881
4882 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4883 WaitForEncodedFrame(CurrentTimeMs());
4884 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4885 VideoLayersAllocation last_layer_allocation =
4886 sink_.GetLastVideoLayersAllocation();
4887
4888 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4889 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4890 .target_bitrate_per_temporal_layer,
4891 SizeIs(1));
4892 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4893 .target_bitrate_per_temporal_layer,
4894 SizeIs(1));
4895 // Since full SVC is used, expect the top layer to utilize the full target
4896 // rate.
4897 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4898 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004899 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004900 video_stream_encoder_->Stop();
4901}
4902
4903TEST_F(VideoStreamEncoderTest,
4904 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4905 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4906 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004907 VideoEncoderConfig video_encoder_config;
4908 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4909 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004910 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004911 video_encoder_config.content_type =
4912 VideoEncoderConfig::ContentType::kRealtimeVideo;
4913 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4914 vp9_settings.numberOfSpatialLayers = 2;
4915 vp9_settings.numberOfTemporalLayers = 2;
4916 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4917 vp9_settings.automaticResizeOn = false;
4918 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004919 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004920 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004921 ConfigureEncoder(std::move(video_encoder_config),
4922 VideoStreamEncoder::BitrateAllocationCallbackType::
4923 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004924
4925 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004926 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004927
4928 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4929 WaitForEncodedFrame(CurrentTimeMs());
4930 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4931 VideoLayersAllocation last_layer_allocation =
4932 sink_.GetLastVideoLayersAllocation();
4933
4934 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4935 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4936 .target_bitrate_per_temporal_layer,
4937 SizeIs(2));
4938 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4939 .target_bitrate_per_temporal_layer,
4940 SizeIs(2));
4941 // Since KSVC is, spatial layers are independend except on key frames.
4942 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4943 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004944 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004945 video_stream_encoder_->Stop();
4946}
4947
4948TEST_F(VideoStreamEncoderTest,
4949 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4950 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4951 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4952 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004953 VideoEncoderConfig video_encoder_config;
4954 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4955 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004956 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004957 video_encoder_config.content_type =
4958 VideoEncoderConfig::ContentType::kRealtimeVideo;
4959 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4960 vp9_settings.numberOfSpatialLayers = 3;
4961 vp9_settings.numberOfTemporalLayers = 2;
4962 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4963 vp9_settings.automaticResizeOn = false;
4964 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004965 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004966 vp9_settings);
4967 // Simulcast layers are used for enabling/disabling streams.
4968 video_encoder_config.simulcast_layers.resize(3);
4969 video_encoder_config.simulcast_layers[0].active = false;
4970 video_encoder_config.simulcast_layers[1].active = true;
4971 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004972 ConfigureEncoder(std::move(video_encoder_config),
4973 VideoStreamEncoder::BitrateAllocationCallbackType::
4974 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004975
4976 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004977 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004978
4979 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4980 WaitForEncodedFrame(CurrentTimeMs());
4981 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4982 VideoLayersAllocation last_layer_allocation =
4983 sink_.GetLastVideoLayersAllocation();
4984
4985 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4986 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4987 .target_bitrate_per_temporal_layer,
4988 SizeIs(2));
4989 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4990 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4991
4992 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4993 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4994 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4995 .target_bitrate_per_temporal_layer,
4996 SizeIs(2));
4997 // Since full SVC is used, expect the top layer to utilize the full target
4998 // rate.
4999 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
5000 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02005001 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005002 video_stream_encoder_->Stop();
5003}
5004
5005TEST_F(VideoStreamEncoderTest,
5006 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
5007 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5008 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5009 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005010 VideoEncoderConfig video_encoder_config;
5011 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5012 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005013 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005014 video_encoder_config.content_type =
5015 VideoEncoderConfig::ContentType::kRealtimeVideo;
5016 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5017 vp9_settings.numberOfSpatialLayers = 3;
5018 vp9_settings.numberOfTemporalLayers = 2;
5019 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5020 vp9_settings.automaticResizeOn = false;
5021 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005022 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005023 vp9_settings);
5024 // Simulcast layers are used for enabling/disabling streams.
5025 video_encoder_config.simulcast_layers.resize(3);
5026 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005027 ConfigureEncoder(std::move(video_encoder_config),
5028 VideoStreamEncoder::BitrateAllocationCallbackType::
5029 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005030
5031 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005032 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005033
5034 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5035 WaitForEncodedFrame(CurrentTimeMs());
5036 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5037 VideoLayersAllocation last_layer_allocation =
5038 sink_.GetLastVideoLayersAllocation();
5039
5040 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
5041 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5042 .target_bitrate_per_temporal_layer,
5043 SizeIs(2));
5044 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
5045 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5046
5047 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
5048 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
5049 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
5050 .target_bitrate_per_temporal_layer,
5051 SizeIs(2));
5052 video_stream_encoder_->Stop();
5053}
5054
5055TEST_F(VideoStreamEncoderTest,
5056 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
5057 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5058 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5059 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005060 VideoEncoderConfig video_encoder_config;
5061 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5062 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005063 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005064 video_encoder_config.content_type =
5065 VideoEncoderConfig::ContentType::kRealtimeVideo;
5066 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5067 vp9_settings.numberOfSpatialLayers = 3;
5068 vp9_settings.numberOfTemporalLayers = 2;
5069 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5070 vp9_settings.automaticResizeOn = false;
5071 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005072 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005073 vp9_settings);
5074 // Simulcast layers are used for enabling/disabling streams.
5075 video_encoder_config.simulcast_layers.resize(3);
5076 video_encoder_config.simulcast_layers[0].active = false;
5077 video_encoder_config.simulcast_layers[1].active = false;
5078 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005079 ConfigureEncoder(std::move(video_encoder_config),
5080 VideoStreamEncoder::BitrateAllocationCallbackType::
5081 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005082
5083 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005084 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005085
5086 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5087 WaitForEncodedFrame(CurrentTimeMs());
5088 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5089 VideoLayersAllocation last_layer_allocation =
5090 sink_.GetLastVideoLayersAllocation();
5091
5092 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5093 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5094 .target_bitrate_per_temporal_layer,
5095 SizeIs(2));
5096 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5097 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5098 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5099 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02005100 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005101 video_stream_encoder_->Stop();
5102}
5103
5104TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5105 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005106 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005107 kVideoLayersAllocation);
5108 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005109 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005110
5111 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5112 WaitForEncodedFrame(CurrentTimeMs());
5113 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5114 VideoLayersAllocation last_layer_allocation =
5115 sink_.GetLastVideoLayersAllocation();
5116
5117 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5118 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5119 .target_bitrate_per_temporal_layer,
5120 SizeIs(1));
5121 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5122 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005123 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005124 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5125 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5126 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5127 video_stream_encoder_->Stop();
5128}
5129
5130TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02005131 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5132 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005133 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02005134 kVideoLayersAllocation);
5135
5136 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005137 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005138
5139 video_source_.IncomingCapturedFrame(
5140 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5141 WaitForEncodedFrame(CurrentTimeMs());
5142 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5143 VideoLayersAllocation last_layer_allocation =
5144 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02005145 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02005146 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5147 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5148 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005149 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 17:53:22 +02005150
5151 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005152 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5153 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005154 video_source_.IncomingCapturedFrame(
5155 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5156 WaitForEncodedFrame(CurrentTimeMs());
5157
5158 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5159 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5160 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5161 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5162 .target_bitrate_per_temporal_layer[0],
5163 DataRate::Zero());
5164
5165 video_stream_encoder_->Stop();
5166}
5167
Per Kjellander4190ce92020-12-15 17:24:55 +01005168TEST_F(VideoStreamEncoderTest,
5169 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5170 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005171 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01005172 kVideoLayersAllocation);
5173
5174 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005175 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5176 0, 0, 0);
Per Kjellander4190ce92020-12-15 17:24:55 +01005177
5178 video_source_.IncomingCapturedFrame(
5179 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5180 WaitForEncodedFrame(CurrentTimeMs());
5181 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5182 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5183 SizeIs(2));
5184 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5185 codec_width_);
5186 EXPECT_EQ(
5187 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5188 codec_height_);
5189
5190 video_source_.IncomingCapturedFrame(
5191 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5192 WaitForEncodedFrame(CurrentTimeMs());
5193 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5194 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5195 SizeIs(2));
5196 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5197 codec_width_ / 2);
5198 EXPECT_EQ(
5199 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5200 codec_height_ / 2);
5201
5202 video_stream_encoder_->Stop();
5203}
5204
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005205TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5206 // 2 TLs configured, temporal layers supported by encoder.
5207 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005208 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005209 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005210 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005211 fake_encoder_.SetTemporalLayersSupported(0, true);
5212
5213 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 10:07:11 +02005214 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005215 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005216 kNumTemporalLayers, /*temporal_id*/ 0,
5217 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005218 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005219 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005220 kNumTemporalLayers, /*temporal_id*/ 1,
5221 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005222 VideoBitrateAllocation expected_bitrate;
5223 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5224 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5225
5226 VerifyAllocatedBitrate(expected_bitrate);
5227 video_stream_encoder_->Stop();
5228}
5229
5230TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5231 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005232 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005233 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005234 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005235 fake_encoder_.SetTemporalLayersSupported(0, false);
5236
5237 // Temporal layers not supported by the encoder.
5238 // Total bitrate should be at ti:0.
5239 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005240 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005241
5242 VerifyAllocatedBitrate(expected_bitrate);
5243 video_stream_encoder_->Stop();
5244}
5245
5246TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005247 webrtc::test::ScopedKeyValueConfig field_trials(
5248 field_trials_,
Per Kjellanderdcef6412020-10-07 15:09:05 +02005249 "WebRTC-Video-QualityScalerSettings/"
5250 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5251 // Reset encoder for field trials to take effect.
5252 ConfigureEncoder(video_encoder_config_.Copy());
5253
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005254 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005255 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005256 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005257 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005258 fake_encoder_.SetTemporalLayersSupported(0, true);
5259 fake_encoder_.SetTemporalLayersSupported(1, false);
5260
5261 const int kS0Bps = 150000;
5262 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005263 kS0Bps *
5264 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5265 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005266 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005267 kS0Bps *
5268 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5269 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005270 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005271 // Temporal layers not supported by si:1.
5272 VideoBitrateAllocation expected_bitrate;
5273 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5274 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5275 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5276
5277 VerifyAllocatedBitrate(expected_bitrate);
5278 video_stream_encoder_->Stop();
5279}
5280
Niels Möller7dc26b72017-12-06 10:27:48 +01005281TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5282 const int kFrameWidth = 1280;
5283 const int kFrameHeight = 720;
5284 const int kFramerate = 24;
5285
Henrik Boström381d1092020-05-12 18:49:07 +02005286 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005287 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005288 test::FrameForwarder source;
5289 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005290 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005291
5292 // Insert a single frame, triggering initial configuration.
5293 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5294 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5295
5296 EXPECT_EQ(
5297 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5298 kDefaultFramerate);
5299
5300 // Trigger reconfigure encoder (without resetting the entire instance).
5301 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005302 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5303 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005304 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005305 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005306 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005307 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5308
5309 // Detector should be updated with fps limit from codec config.
5310 EXPECT_EQ(
5311 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5312 kFramerate);
5313
5314 // Trigger overuse, max framerate should be reduced.
5315 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5316 stats.input_frame_rate = kFramerate;
5317 stats_proxy_->SetMockStats(stats);
5318 video_stream_encoder_->TriggerCpuOveruse();
5319 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5320 int adapted_framerate =
5321 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5322 EXPECT_LT(adapted_framerate, kFramerate);
5323
5324 // Trigger underuse, max framerate should go back to codec configured fps.
5325 // Set extra low fps, to make sure it's actually reset, not just incremented.
5326 stats = stats_proxy_->GetStats();
5327 stats.input_frame_rate = adapted_framerate / 2;
5328 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005329 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005330 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5331 EXPECT_EQ(
5332 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5333 kFramerate);
5334
5335 video_stream_encoder_->Stop();
5336}
5337
5338TEST_F(VideoStreamEncoderTest,
5339 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5340 const int kFrameWidth = 1280;
5341 const int kFrameHeight = 720;
5342 const int kLowFramerate = 15;
5343 const int kHighFramerate = 25;
5344
Henrik Boström381d1092020-05-12 18:49:07 +02005345 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005346 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005347 test::FrameForwarder source;
5348 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005349 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005350
5351 // Trigger initial configuration.
5352 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005353 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5354 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005355 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005356 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005357 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005358 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005359 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5360
5361 EXPECT_EQ(
5362 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5363 kLowFramerate);
5364
5365 // Trigger overuse, max framerate should be reduced.
5366 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5367 stats.input_frame_rate = kLowFramerate;
5368 stats_proxy_->SetMockStats(stats);
5369 video_stream_encoder_->TriggerCpuOveruse();
5370 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5371 int adapted_framerate =
5372 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5373 EXPECT_LT(adapted_framerate, kLowFramerate);
5374
5375 // Reconfigure the encoder with a new (higher max framerate), max fps should
5376 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005377 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005378 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5379 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005380 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005381 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5382
5383 EXPECT_EQ(
5384 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5385 adapted_framerate);
5386
5387 // Trigger underuse, max framerate should go back to codec configured fps.
5388 stats = stats_proxy_->GetStats();
5389 stats.input_frame_rate = adapted_framerate;
5390 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005391 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005392 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5393 EXPECT_EQ(
5394 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5395 kHighFramerate);
5396
5397 video_stream_encoder_->Stop();
5398}
5399
mflodmancc3d4422017-08-03 08:27:51 -07005400TEST_F(VideoStreamEncoderTest,
5401 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005402 const int kFrameWidth = 1280;
5403 const int kFrameHeight = 720;
5404 const int kFramerate = 24;
5405
Henrik Boström381d1092020-05-12 18:49:07 +02005406 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005407 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005408 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005409 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005410 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005411
5412 // Trigger initial configuration.
5413 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005414 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5415 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005416 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 04:21:07 -07005417 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005418 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005419 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005420 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005421
Niels Möller7dc26b72017-12-06 10:27:48 +01005422 EXPECT_EQ(
5423 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5424 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005425
5426 // Trigger overuse, max framerate should be reduced.
5427 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5428 stats.input_frame_rate = kFramerate;
5429 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005430 video_stream_encoder_->TriggerCpuOveruse();
5431 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005432 int adapted_framerate =
5433 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005434 EXPECT_LT(adapted_framerate, kFramerate);
5435
5436 // Change degradation preference to not enable framerate scaling. Target
5437 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005438 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005439 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005440 EXPECT_EQ(
5441 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5442 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005443
mflodmancc3d4422017-08-03 08:27:51 -07005444 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005445}
5446
mflodmancc3d4422017-08-03 08:27:51 -07005447TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005448 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005449 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005450 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5451 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5452 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005453 const int kWidth = 640;
5454 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005455
asaperssonfab67072017-04-04 05:51:49 -07005456 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005457
5458 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005459 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005460
5461 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005462 EXPECT_TRUE_WAIT(
5463 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005464
sprangc5d62e22017-04-02 23:53:04 -07005465 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005466
asaperssonfab67072017-04-04 05:51:49 -07005467 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005468 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005469 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005470
5471 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005472 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005473
Henrik Boström2671dac2020-05-19 16:29:09 +02005474 EXPECT_TRUE_WAIT(
5475 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005476
mflodmancc3d4422017-08-03 08:27:51 -07005477 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005478}
5479
mflodmancc3d4422017-08-03 08:27:51 -07005480TEST_F(VideoStreamEncoderTest,
5481 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005482 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005483 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005484 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5485 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5486 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005487 const int kWidth = 640;
5488 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005489
5490 // We expect the n initial frames to get dropped.
5491 int i;
5492 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005493 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005494 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005495 }
5496 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005497 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005498 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005499
5500 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005501 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005502
mflodmancc3d4422017-08-03 08:27:51 -07005503 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005504}
5505
mflodmancc3d4422017-08-03 08:27:51 -07005506TEST_F(VideoStreamEncoderTest,
5507 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005508 const int kWidth = 640;
5509 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005510 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005511 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005512
5513 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005514 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005515 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005516
asaperssonfab67072017-04-04 05:51:49 -07005517 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005518 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005519 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005520
mflodmancc3d4422017-08-03 08:27:51 -07005521 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005522}
5523
mflodmancc3d4422017-08-03 08:27:51 -07005524TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005525 const int kWidth = 640;
5526 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005527 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005528
5529 VideoEncoderConfig video_encoder_config;
5530 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5531 // Make format different, to force recreation of encoder.
5532 video_encoder_config.video_format.parameters["foo"] = "foo";
5533 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005534 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005535 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005536 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005537
kthelgasonb83797b2017-02-14 11:57:25 -08005538 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005539 video_stream_encoder_->SetSource(&video_source_,
5540 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005541
asaperssonfab67072017-04-04 05:51:49 -07005542 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005543 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005544 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005545
mflodmancc3d4422017-08-03 08:27:51 -07005546 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005547 fake_encoder_.SetQualityScaling(true);
5548}
5549
Åsa Persson139f4dc2019-08-02 09:29:58 +02005550TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005551 webrtc::test::ScopedKeyValueConfig field_trials(
5552 field_trials_,
Åsa Persson139f4dc2019-08-02 09:29:58 +02005553 "WebRTC-Video-QualityScalerSettings/"
5554 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5555 // Reset encoder for field trials to take effect.
5556 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005557 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5558 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 09:29:58 +02005559 const int kWidth = 640;
5560 const int kHeight = 360;
5561
Henrik Boström381d1092020-05-12 18:49:07 +02005562 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005563 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005564 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5565 // Frame should not be dropped.
5566 WaitForEncodedFrame(1);
5567
Henrik Boström381d1092020-05-12 18:49:07 +02005568 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005569 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5570 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5571 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005572 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5573 // Frame should not be dropped.
5574 WaitForEncodedFrame(2);
5575
Henrik Boström381d1092020-05-12 18:49:07 +02005576 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005577 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5578 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5579 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005580 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5581 // Expect to drop this frame, the wait should time out.
5582 ExpectDroppedFrame();
5583
5584 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005585 EXPECT_TRUE_WAIT(
5586 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005587 video_stream_encoder_->Stop();
5588}
5589
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005590TEST_F(VideoStreamEncoderTest,
5591 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005592 webrtc::test::ScopedKeyValueConfig field_trials(
5593 field_trials_,
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005594 "WebRTC-Video-QualityScalerSettings/"
5595 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5596 fake_encoder_.SetQualityScaling(false);
5597 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005598 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5599 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005600 const int kWidth = 640;
5601 const int kHeight = 360;
5602
5603 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005604 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005605 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5606 // Frame should not be dropped.
5607 WaitForEncodedFrame(1);
5608
5609 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5610 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5611 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5612 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5613 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5614 // Frame should not be dropped.
5615 WaitForEncodedFrame(2);
5616
5617 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5618 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5619 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5620 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5621 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5622 // Not dropped since quality scaling is disabled.
5623 WaitForEncodedFrame(3);
5624
5625 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005626 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005627 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5628
5629 video_stream_encoder_->Stop();
5630}
5631
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005632TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005633 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005634 // Set simulcast.
5635 ResetEncoder("VP8", 3, 1, 1, false);
5636 fake_encoder_.SetQualityScaling(true);
5637 const int kWidth = 1280;
5638 const int kHeight = 720;
5639 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005640 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005641 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5642 // Frame should not be dropped.
5643 WaitForEncodedFrame(1);
5644
5645 // Trigger QVGA "singlecast"
5646 // Update the config.
5647 VideoEncoderConfig video_encoder_config;
5648 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5649 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005650 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005651 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005652 "VP8", /*max qp*/ 56, /*screencast*/ false,
5653 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005654 for (auto& layer : video_encoder_config.simulcast_layers) {
5655 layer.num_temporal_layers = 1;
5656 layer.max_framerate = kDefaultFramerate;
5657 }
Asa Persson606d3cb2021-10-04 10:07:11 +02005658 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005659 video_encoder_config.content_type =
5660 VideoEncoderConfig::ContentType::kRealtimeVideo;
5661
5662 video_encoder_config.simulcast_layers[0].active = true;
5663 video_encoder_config.simulcast_layers[1].active = false;
5664 video_encoder_config.simulcast_layers[2].active = false;
5665
5666 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5667 kMaxPayloadLength);
5668 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5669
5670 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5671 // Frame should not be dropped.
5672 WaitForEncodedFrame(2);
5673
5674 // Trigger HD "singlecast"
5675 video_encoder_config.simulcast_layers[0].active = false;
5676 video_encoder_config.simulcast_layers[1].active = false;
5677 video_encoder_config.simulcast_layers[2].active = true;
5678
5679 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5680 kMaxPayloadLength);
5681 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5682
5683 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5684 // Frame should be dropped because of initial frame drop.
5685 ExpectDroppedFrame();
5686
5687 // Expect the sink_wants to specify a scaled frame.
5688 EXPECT_TRUE_WAIT(
5689 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5690 video_stream_encoder_->Stop();
5691}
5692
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005693TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005694 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005695 // Set simulcast.
5696 ResetEncoder("VP9", 1, 1, 3, false);
5697 fake_encoder_.SetQualityScaling(true);
5698 const int kWidth = 1280;
5699 const int kHeight = 720;
5700 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005701 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005702 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5703 // Frame should not be dropped.
5704 WaitForEncodedFrame(1);
5705
5706 // Trigger QVGA "singlecast"
5707 // Update the config.
5708 VideoEncoderConfig video_encoder_config;
5709 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5710 &video_encoder_config);
5711 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5712 vp9_settings.numberOfSpatialLayers = 3;
5713 // Since only one layer is active - automatic resize should be enabled.
5714 vp9_settings.automaticResizeOn = true;
5715 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005716 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005717 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005718 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005719 video_encoder_config.content_type =
5720 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 16:22:11 +02005721 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005722 // which SVC layers are active.
5723 video_encoder_config.simulcast_layers.resize(3);
5724
5725 video_encoder_config.simulcast_layers[0].active = true;
5726 video_encoder_config.simulcast_layers[1].active = false;
5727 video_encoder_config.simulcast_layers[2].active = false;
5728
5729 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5730 kMaxPayloadLength);
5731 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5732
5733 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5734 // Frame should not be dropped.
5735 WaitForEncodedFrame(2);
5736
5737 // Trigger HD "singlecast"
5738 video_encoder_config.simulcast_layers[0].active = false;
5739 video_encoder_config.simulcast_layers[1].active = false;
5740 video_encoder_config.simulcast_layers[2].active = true;
5741
5742 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5743 kMaxPayloadLength);
5744 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5745
5746 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5747 // Frame should be dropped because of initial frame drop.
5748 ExpectDroppedFrame();
5749
5750 // Expect the sink_wants to specify a scaled frame.
5751 EXPECT_TRUE_WAIT(
5752 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5753 video_stream_encoder_->Stop();
5754}
5755
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005756TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005757 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5758 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5759 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5760 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5761 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5762 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5763 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5764 fake_encoder_.SetResolutionBitrateLimits(
5765 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5766
5767 VideoEncoderConfig video_encoder_config;
5768 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5769 &video_encoder_config);
5770 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5771 vp9_settings.numberOfSpatialLayers = 3;
5772 // Since only one layer is active - automatic resize should be enabled.
5773 vp9_settings.automaticResizeOn = true;
5774 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005775 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005776 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005777 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005778 video_encoder_config.content_type =
5779 VideoEncoderConfig::ContentType::kRealtimeVideo;
5780 // Simulcast layers are used to indicate which spatial layers are active.
5781 video_encoder_config.simulcast_layers.resize(3);
5782 video_encoder_config.simulcast_layers[0].active = false;
5783 video_encoder_config.simulcast_layers[1].active = true;
5784 video_encoder_config.simulcast_layers[2].active = false;
5785
5786 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5787 kMaxPayloadLength);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005788
5789 // The encoder bitrate limits for 360p should be used.
5790 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005791 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005792 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5793 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5794 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5795 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5796 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5797 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005798 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005799 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005800 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005801 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005802
5803 // The encoder bitrate limits for 270p should be used.
5804 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02005805 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005806 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5807 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5808 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5809 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5810 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5811 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005812 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005813 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005814 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005815 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005816
5817 video_stream_encoder_->Stop();
5818}
5819
5820TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005821 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5822 VideoEncoderConfig video_encoder_config;
5823 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5824 &video_encoder_config);
5825 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5826 vp9_settings.numberOfSpatialLayers = 3;
5827 // Since only one layer is active - automatic resize should be enabled.
5828 vp9_settings.automaticResizeOn = true;
5829 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005830 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005831 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005832 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005833 video_encoder_config.content_type =
5834 VideoEncoderConfig::ContentType::kRealtimeVideo;
5835 // Simulcast layers are used to indicate which spatial layers are active.
5836 video_encoder_config.simulcast_layers.resize(3);
5837 video_encoder_config.simulcast_layers[0].active = false;
5838 video_encoder_config.simulcast_layers[1].active = true;
5839 video_encoder_config.simulcast_layers[2].active = false;
5840
5841 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5842 kMaxPayloadLength);
Åsa Persson258e9892021-02-25 10:39:51 +01005843
5844 // The default bitrate limits for 360p should be used.
5845 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005846 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5847 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005848 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005849 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005850 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5851 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5852 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5853 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5854 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5855 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005856 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005857 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005858 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005859 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005860
5861 // The default bitrate limits for 270p should be used.
5862 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005863 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5864 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005865 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02005866 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005867 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5868 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5869 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5870 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5871 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5872 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005873 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005874 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005875 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005876 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005877
5878 video_stream_encoder_->Stop();
5879}
5880
5881TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005882 webrtc::test::ScopedKeyValueConfig field_trials(
5883 field_trials_, "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
Åsa Persson258e9892021-02-25 10:39:51 +01005884 VideoEncoderConfig video_encoder_config;
5885 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5886 &video_encoder_config);
5887 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5888 vp9_settings.numberOfSpatialLayers = 3;
5889 // Since only one layer is active - automatic resize should be enabled.
5890 vp9_settings.automaticResizeOn = true;
5891 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005892 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005893 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005894 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005895 video_encoder_config.content_type =
5896 VideoEncoderConfig::ContentType::kRealtimeVideo;
5897 // Simulcast layers are used to indicate which spatial layers are active.
5898 video_encoder_config.simulcast_layers.resize(3);
5899 video_encoder_config.simulcast_layers[0].active = false;
5900 video_encoder_config.simulcast_layers[1].active = true;
5901 video_encoder_config.simulcast_layers[2].active = false;
5902
5903 // Reset encoder for field trials to take effect.
5904 ConfigureEncoder(video_encoder_config.Copy());
5905
5906 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5907 kMaxPayloadLength);
Åsa Persson258e9892021-02-25 10:39:51 +01005908
5909 // The default bitrate limits for 360p should not be used.
5910 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005911 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5912 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005913 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005914 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005915 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5916 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5917 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5918 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5919 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5920 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005921 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005922 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005923
5924 video_stream_encoder_->Stop();
5925}
5926
5927TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5928 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5929 /*num_spatial_layers=*/1, /*screenshare=*/false);
5930
5931 // The default singlecast bitrate limits for 720p should not be used.
5932 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005933 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5934 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005935 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005936 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005937 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5938 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5939 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5940 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5941 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5942 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005943 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005944 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005945
5946 video_stream_encoder_->Stop();
5947}
5948
5949TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005950 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5951 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5952 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5953 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5954 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5955 fake_encoder_.SetResolutionBitrateLimits(
5956 {kEncoderLimits180p, kEncoderLimits720p});
5957
5958 VideoEncoderConfig video_encoder_config;
5959 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5960 &video_encoder_config);
5961 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5962 vp9_settings.numberOfSpatialLayers = 3;
5963 // Since only one layer is active - automatic resize should be enabled.
5964 vp9_settings.automaticResizeOn = true;
5965 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005966 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005967 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005968 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005969 video_encoder_config.content_type =
5970 VideoEncoderConfig::ContentType::kRealtimeVideo;
5971 // Simulcast layers are used to indicate which spatial layers are active.
5972 video_encoder_config.simulcast_layers.resize(3);
5973 video_encoder_config.simulcast_layers[0].active = true;
5974 video_encoder_config.simulcast_layers[1].active = false;
5975 video_encoder_config.simulcast_layers[2].active = false;
5976
5977 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5978 kMaxPayloadLength);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005979
5980 // Limits not applied on lowest stream, limits for 180p should not be used.
5981 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005982 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005983 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5984 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5985 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5986 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5987 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5988 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005989 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005990 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005991 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005992 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005993
5994 video_stream_encoder_->Stop();
5995}
5996
5997TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005998 InitialFrameDropActivatesWhenResolutionIncreases) {
5999 const int kWidth = 640;
6000 const int kHeight = 360;
6001
6002 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006003 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006004 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
6005 // Frame should not be dropped.
6006 WaitForEncodedFrame(1);
6007
6008 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006009 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006010 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
6011 // Frame should not be dropped, bitrate not too low for frame.
6012 WaitForEncodedFrame(2);
6013
6014 // Incoming resolution increases.
6015 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
6016 // Expect to drop this frame, bitrate too low for frame.
6017 ExpectDroppedFrame();
6018
6019 // Expect the sink_wants to specify a scaled frame.
6020 EXPECT_TRUE_WAIT(
6021 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
6022 video_stream_encoder_->Stop();
6023}
6024
6025TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
6026 const int kWidth = 640;
6027 const int kHeight = 360;
6028 // So that quality scaling doesn't happen by itself.
6029 fake_encoder_.SetQp(kQpHigh);
6030
6031 AdaptingFrameForwarder source(&time_controller_);
6032 source.set_adaptation_enabled(true);
6033 video_stream_encoder_->SetSource(
6034 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
6035
6036 int timestamp = 1;
6037
6038 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006039 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006040 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6041 WaitForEncodedFrame(timestamp);
6042 timestamp += 9000;
6043 // Long pause to disable all first BWE drop logic.
6044 AdvanceTime(TimeDelta::Millis(1000));
6045
6046 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006047 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006048 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6049 // Not dropped frame, as initial frame drop is disabled by now.
6050 WaitForEncodedFrame(timestamp);
6051 timestamp += 9000;
6052 AdvanceTime(TimeDelta::Millis(100));
6053
6054 // Quality adaptation down.
6055 video_stream_encoder_->TriggerQualityLow();
6056
6057 // Adaptation has an effect.
6058 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6059 5000);
6060
6061 // Frame isn't dropped as initial frame dropper is disabled.
6062 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6063 WaitForEncodedFrame(timestamp);
6064 timestamp += 9000;
6065 AdvanceTime(TimeDelta::Millis(100));
6066
6067 // Quality adaptation up.
6068 video_stream_encoder_->TriggerQualityHigh();
6069
6070 // Adaptation has an effect.
6071 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
6072 5000);
6073
6074 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6075 // Frame should not be dropped, as initial framedropper is off.
6076 WaitForEncodedFrame(timestamp);
6077
6078 video_stream_encoder_->Stop();
6079}
6080
Åsa Persson7f354f82021-02-04 15:52:15 +01006081TEST_F(VideoStreamEncoderTest,
6082 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
6083 const int kMinStartBps360p = 222000;
6084 fake_encoder_.SetResolutionBitrateLimits(
6085 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6086 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6087 800000)});
6088
6089 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6090 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6091 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6092 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
6093 0, 0, 0);
6094 // Frame should not be dropped, bitrate not too low for frame.
6095 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6096 WaitForEncodedFrame(1);
6097
6098 // Incoming resolution increases, initial frame drop activates.
6099 // Frame should be dropped, link allocation too low for frame.
6100 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6101 ExpectDroppedFrame();
6102
6103 // Expect sink_wants to specify a scaled frame.
6104 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6105 5000);
6106 video_stream_encoder_->Stop();
6107}
6108
6109TEST_F(VideoStreamEncoderTest,
6110 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6111 const int kMinStartBps360p = 222000;
6112 fake_encoder_.SetResolutionBitrateLimits(
6113 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6114 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6115 800000)});
6116
6117 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6118 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6119 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6120 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6121 0, 0, 0);
6122 // Frame should not be dropped, bitrate not too low for frame.
6123 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6124 WaitForEncodedFrame(1);
6125
6126 // Incoming resolution increases, initial frame drop activates.
6127 // Frame should be dropped, link allocation not too low for frame.
6128 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6129 WaitForEncodedFrame(2);
6130
6131 video_stream_encoder_->Stop();
6132}
6133
Åsa Perssone644a032019-11-08 15:56:00 +01006134TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006135 webrtc::test::ScopedKeyValueConfig field_trials(
6136 field_trials_,
Åsa Persson06defc42021-09-10 15:28:48 +02006137 "WebRTC-Video-QualityRampupSettings/"
6138 "min_pixels:921600,min_duration_ms:2000/");
6139
6140 const int kWidth = 1280;
6141 const int kHeight = 720;
6142 const int kFps = 10;
6143 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006144
6145 // Reset encoder for field trials to take effect.
6146 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 10:07:11 +02006147 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 11:34:32 +02006148 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01006149 ConfigureEncoder(std::move(config));
6150 fake_encoder_.SetQp(kQpLow);
6151
6152 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006153 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01006154 source.set_adaptation_enabled(true);
6155 video_stream_encoder_->SetSource(&source,
6156 DegradationPreference::MAINTAIN_FRAMERATE);
6157
6158 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 10:07:11 +02006159 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 18:49:07 +02006160 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006161 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006162
6163 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 15:28:48 +02006164 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006165 int64_t timestamp_ms = kFrameIntervalMs;
6166 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6167 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02006168 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6169 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01006170
6171 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02006172 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6173 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006174
Artem Titovab30d722021-07-27 16:22:11 +02006175 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 15:28:48 +02006176 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 15:56:00 +01006177 for (size_t i = 1; i <= 10; i++) {
6178 timestamp_ms += kFrameIntervalMs;
6179 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6180 WaitForEncodedFrame(timestamp_ms);
6181 }
Åsa Persson06defc42021-09-10 15:28:48 +02006182
6183 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6184 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6185 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6186 timestamp_ms += kFrameIntervalMs;
6187 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6188 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 15:56:00 +01006189 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6190 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6191
Åsa Persson06defc42021-09-10 15:28:48 +02006192 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 15:56:00 +01006193 timestamp_ms += kFrameIntervalMs;
6194 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6195 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006196 // The ramp-up code involves the adaptation queue, give it time to execute.
6197 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006198 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006199 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006200
6201 // Frame should not be adapted.
6202 timestamp_ms += kFrameIntervalMs;
6203 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6204 WaitForEncodedFrame(kWidth, kHeight);
6205 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6206
6207 video_stream_encoder_->Stop();
6208}
6209
mflodmancc3d4422017-08-03 08:27:51 -07006210TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006211 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006212 webrtc::test::ScopedKeyValueConfig field_trials(
6213 field_trials_, "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006214 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006215 source.set_adaptation_enabled(true);
6216 video_stream_encoder_->SetSource(&source,
6217 DegradationPreference::MAINTAIN_FRAMERATE);
6218 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006219 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006220 fake_encoder_.SetQp(kQpHigh + 1);
6221 const int kWidth = 1280;
6222 const int kHeight = 720;
6223 const int64_t kFrameIntervalMs = 100;
6224 int64_t timestamp_ms = kFrameIntervalMs;
6225 for (size_t i = 1; i <= 100; i++) {
6226 timestamp_ms += kFrameIntervalMs;
6227 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6228 WaitForEncodedFrame(timestamp_ms);
6229 }
6230 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6231 // for the first time.
6232 // TODO(eshr): We should avoid these waits by using threads with simulated
6233 // time.
6234 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6235 2000 * 2.5 * 2);
6236 timestamp_ms += kFrameIntervalMs;
6237 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6238 WaitForEncodedFrame(timestamp_ms);
6239 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6240 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6241 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6242
6243 // Disable Quality scaling by turning off scaler on the encoder and
6244 // reconfiguring.
6245 fake_encoder_.SetQualityScaling(false);
6246 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6247 kMaxPayloadLength);
6248 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Markus Handell28c71802021-11-08 10:11:55 +01006249 AdvanceTime(TimeDelta::Zero());
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006250 // Since we turned off the quality scaler, the adaptations made by it are
6251 // removed.
6252 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6253 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6254
6255 video_stream_encoder_->Stop();
6256}
6257
6258TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006259 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6260 const int kTooSmallWidth = 10;
6261 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006262 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006263 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006264
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006265 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006266 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006267 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006268 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006269 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006270 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6271
6272 // Trigger adapt down, too small frame, expect no change.
6273 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006274 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006275 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006276 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006277 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6278 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6279
mflodmancc3d4422017-08-03 08:27:51 -07006280 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006281}
6282
mflodmancc3d4422017-08-03 08:27:51 -07006283TEST_F(VideoStreamEncoderTest,
6284 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006285 const int kTooSmallWidth = 10;
6286 const int kTooSmallHeight = 10;
6287 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006288 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006289 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006290
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006291 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006292 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006293 video_stream_encoder_->SetSource(&source,
6294 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006295 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006296 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6297 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6298
6299 // Trigger adapt down, expect limited framerate.
6300 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006301 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006302 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006303 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006304 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6305 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6306 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6307
6308 // Trigger adapt down, too small frame, expect no change.
6309 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006310 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006311 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006312 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006313 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6314 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6315 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6316
mflodmancc3d4422017-08-03 08:27:51 -07006317 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006318}
6319
mflodmancc3d4422017-08-03 08:27:51 -07006320TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006321 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006322 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006323 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006324 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006325 const int kFrameWidth = 1280;
6326 const int kFrameHeight = 720;
6327 video_source_.IncomingCapturedFrame(
6328 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006329 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006330 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006331}
6332
sprangb1ca0732017-02-01 08:38:12 -08006333// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006334TEST_F(VideoStreamEncoderTest,
6335 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006336 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006337 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006338
6339 const int kFrameWidth = 1280;
6340 const int kFrameHeight = 720;
6341 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006342 // requested by
6343 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006344 video_source_.set_adaptation_enabled(true);
6345
6346 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006347 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006348 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006349
6350 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006351 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006352 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006353 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006354 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006355
asaperssonfab67072017-04-04 05:51:49 -07006356 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006357 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006358 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006359 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006360 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006361
mflodmancc3d4422017-08-03 08:27:51 -07006362 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006363}
sprangfe627f32017-03-29 08:24:59 -07006364
mflodmancc3d4422017-08-03 08:27:51 -07006365TEST_F(VideoStreamEncoderTest,
6366 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006367 const int kFrameWidth = 1280;
6368 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006369
Henrik Boström381d1092020-05-12 18:49:07 +02006370 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006371 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006372 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006373 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006374 video_source_.set_adaptation_enabled(true);
6375
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006376 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006377
6378 video_source_.IncomingCapturedFrame(
6379 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006380 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006381
6382 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006383 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006384
6385 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006386 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006387 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006388 video_source_.IncomingCapturedFrame(
6389 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006390 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006391 }
6392
6393 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006394 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006395 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006396 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006397 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006398 video_source_.IncomingCapturedFrame(
6399 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Markus Handell2cfc1af2022-08-19 08:16:48 +00006400 if (!WaitForFrame(kFrameTimeout)) {
sprangc5d62e22017-04-02 23:53:04 -07006401 ++num_frames_dropped;
6402 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006403 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006404 }
6405 }
6406
sprang4847ae62017-06-27 07:06:52 -07006407 // Add some slack to account for frames dropped by the frame dropper.
6408 const int kErrorMargin = 1;
6409 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006410 kErrorMargin);
6411
6412 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006413 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006414 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006415 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006416 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006417 video_source_.IncomingCapturedFrame(
6418 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Markus Handell2cfc1af2022-08-19 08:16:48 +00006419 if (!WaitForFrame(kFrameTimeout)) {
sprangc5d62e22017-04-02 23:53:04 -07006420 ++num_frames_dropped;
6421 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006422 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006423 }
6424 }
sprang4847ae62017-06-27 07:06:52 -07006425 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006426 kErrorMargin);
6427
6428 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006429 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006430 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006431 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006432 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006433 video_source_.IncomingCapturedFrame(
6434 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Markus Handell2cfc1af2022-08-19 08:16:48 +00006435 if (!WaitForFrame(kFrameTimeout)) {
sprangc5d62e22017-04-02 23:53:04 -07006436 ++num_frames_dropped;
6437 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006438 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006439 }
6440 }
sprang4847ae62017-06-27 07:06:52 -07006441 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006442 kErrorMargin);
6443
6444 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006445 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006446 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006447 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006448 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006449 video_source_.IncomingCapturedFrame(
6450 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Markus Handell2cfc1af2022-08-19 08:16:48 +00006451 if (!WaitForFrame(kFrameTimeout)) {
sprangc5d62e22017-04-02 23:53:04 -07006452 ++num_frames_dropped;
6453 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006454 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006455 }
6456 }
6457 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6458
mflodmancc3d4422017-08-03 08:27:51 -07006459 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006460}
6461
mflodmancc3d4422017-08-03 08:27:51 -07006462TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006463 const int kFramerateFps = 5;
6464 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006465 const int kFrameWidth = 1280;
6466 const int kFrameHeight = 720;
6467
sprang4847ae62017-06-27 07:06:52 -07006468 // Reconfigure encoder with two temporal layers and screensharing, which will
6469 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006470 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006471
Henrik Boström381d1092020-05-12 18:49:07 +02006472 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006473 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006474 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006475 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006476 video_source_.set_adaptation_enabled(true);
6477
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006478 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006479
6480 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006481 rtc::VideoSinkWants last_wants;
6482 do {
6483 last_wants = video_source_.sink_wants();
6484
sprangc5d62e22017-04-02 23:53:04 -07006485 // Insert frames to get a new fps estimate...
6486 for (int j = 0; j < kFramerateFps; ++j) {
6487 video_source_.IncomingCapturedFrame(
6488 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006489 if (video_source_.last_sent_width()) {
6490 sink_.WaitForEncodedFrame(timestamp_ms);
6491 }
sprangc5d62e22017-04-02 23:53:04 -07006492 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006493 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006494 }
6495 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006496 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006497 } while (video_source_.sink_wants().max_framerate_fps <
6498 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006499
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006500 EXPECT_THAT(video_source_.sink_wants(),
6501 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006502
mflodmancc3d4422017-08-03 08:27:51 -07006503 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006504}
asaperssonf7e294d2017-06-13 23:25:22 -07006505
mflodmancc3d4422017-08-03 08:27:51 -07006506TEST_F(VideoStreamEncoderTest,
6507 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006508 const int kWidth = 1280;
6509 const int kHeight = 720;
6510 const int64_t kFrameIntervalMs = 150;
6511 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006512 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006513 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006514
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006515 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006516 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006517 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006518 video_stream_encoder_->SetSource(&source,
6519 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006520 timestamp_ms += kFrameIntervalMs;
6521 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006522 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006523 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006524 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6525 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6526 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6527
6528 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006529 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006530 timestamp_ms += kFrameIntervalMs;
6531 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006532 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006533 EXPECT_THAT(source.sink_wants(),
6534 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006535 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6536 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6537 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6538
6539 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006540 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006541 timestamp_ms += kFrameIntervalMs;
6542 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006543 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006544 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006545 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6546 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6547 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6548
6549 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006550 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006551 timestamp_ms += kFrameIntervalMs;
6552 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006553 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006554 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006555 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6556 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6557 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6558
6559 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006560 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006561 timestamp_ms += kFrameIntervalMs;
6562 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006563 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006564 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006565 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6566 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6567 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6568
6569 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006570 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006571 timestamp_ms += kFrameIntervalMs;
6572 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006573 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006574 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006575 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6576 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6577 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6578
6579 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006580 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006581 timestamp_ms += kFrameIntervalMs;
6582 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006583 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006584 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006585 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6586 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6587 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6588
6589 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006590 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006591 timestamp_ms += kFrameIntervalMs;
6592 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006593 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006594 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006595 rtc::VideoSinkWants last_wants = source.sink_wants();
6596 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6597 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6598 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6599
6600 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006601 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006602 timestamp_ms += kFrameIntervalMs;
6603 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006604 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006605 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006606 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6607 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6608 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6609
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006610 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006611 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006612 timestamp_ms += kFrameIntervalMs;
6613 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006614 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006615 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006616 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6617 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6618 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6619
6620 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006621 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006622 timestamp_ms += kFrameIntervalMs;
6623 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006624 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006625 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006626 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6627 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6628 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6629
6630 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006631 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006632 timestamp_ms += kFrameIntervalMs;
6633 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006634 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006635 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006636 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6637 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6638 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6639
6640 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006641 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006642 timestamp_ms += kFrameIntervalMs;
6643 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006644 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006645 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006646 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6647 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6648 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6649
6650 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006651 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006652 timestamp_ms += kFrameIntervalMs;
6653 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006654 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006655 EXPECT_THAT(source.sink_wants(), FpsMax());
6656 EXPECT_EQ(source.sink_wants().max_pixel_count,
6657 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006658 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6659 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6660 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6661
6662 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006663 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006664 timestamp_ms += kFrameIntervalMs;
6665 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006666 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006667 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006668 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6669 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6670 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6671
Åsa Persson30ab0152019-08-27 12:22:33 +02006672 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006673 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006674 timestamp_ms += kFrameIntervalMs;
6675 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006676 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006677 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006678 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006679 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6680 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6681 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6682
6683 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006684 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006685 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006686 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6687
mflodmancc3d4422017-08-03 08:27:51 -07006688 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006689}
6690
mflodmancc3d4422017-08-03 08:27:51 -07006691TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006692 const int kWidth = 1280;
6693 const int kHeight = 720;
6694 const int64_t kFrameIntervalMs = 150;
6695 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006696 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006697 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006698
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006699 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006700 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006701 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006702 video_stream_encoder_->SetSource(&source,
6703 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006704 timestamp_ms += kFrameIntervalMs;
6705 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006706 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006707 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006708 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6709 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6710 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6711 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6712 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6713 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6714
6715 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006716 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006717 timestamp_ms += kFrameIntervalMs;
6718 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006719 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006720 EXPECT_THAT(source.sink_wants(),
6721 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006722 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6723 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6724 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6725 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6726 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6727 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6728
6729 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006730 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006731 timestamp_ms += kFrameIntervalMs;
6732 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006733 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006734 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006735 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6736 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6737 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6738 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6739 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6740 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6741
6742 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006743 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006744 timestamp_ms += kFrameIntervalMs;
6745 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006746 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006747 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006748 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006749 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6750 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6751 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6752 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6753 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6754
Evan Shrubsole64469032020-06-11 10:45:29 +02006755 // Trigger cpu adapt up, expect no change since QP is most limited.
6756 {
6757 // Store current sink wants since we expect no change and if there is no
6758 // change then last_wants() is not updated.
6759 auto previous_sink_wants = source.sink_wants();
6760 video_stream_encoder_->TriggerCpuUnderuse();
6761 timestamp_ms += kFrameIntervalMs;
6762 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6763 WaitForEncodedFrame(timestamp_ms);
6764 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6765 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6766 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6767 }
6768
6769 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6770 video_stream_encoder_->TriggerQualityHigh();
6771 timestamp_ms += kFrameIntervalMs;
6772 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6773 WaitForEncodedFrame(timestamp_ms);
6774 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6775 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6776 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6777 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6778 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6779 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6780 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6781
6782 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6783 // expect increased resolution (960x540@30fps).
6784 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006785 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006786 timestamp_ms += kFrameIntervalMs;
6787 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006788 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006789 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006790 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6791 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6792 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6793 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6794 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006795 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006796
Evan Shrubsole64469032020-06-11 10:45:29 +02006797 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6798 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006799 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006800 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006801 timestamp_ms += kFrameIntervalMs;
6802 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006803 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006804 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006805 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006806 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6807 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6808 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6809 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6810 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006811 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006812
6813 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006814 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006815 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006816 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006817 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006818
mflodmancc3d4422017-08-03 08:27:51 -07006819 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006820}
6821
mflodmancc3d4422017-08-03 08:27:51 -07006822TEST_F(VideoStreamEncoderTest,
6823 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006824 const int kWidth = 640;
6825 const int kHeight = 360;
6826 const int kFpsLimit = 15;
6827 const int64_t kFrameIntervalMs = 150;
6828 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006829 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006830 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006831
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006832 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006833 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006834 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006835 video_stream_encoder_->SetSource(&source,
6836 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006837 timestamp_ms += kFrameIntervalMs;
6838 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006839 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006840 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006841 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6842 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6843 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6844 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6845 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6846 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6847
6848 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006849 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006850 timestamp_ms += kFrameIntervalMs;
6851 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006852 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006853 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006854 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6855 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6856 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6857 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6858 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6859 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6860
6861 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006862 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006863 timestamp_ms += kFrameIntervalMs;
6864 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006865 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006866 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006867 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006868 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006869 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6870 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6871 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6872 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6873
Evan Shrubsole64469032020-06-11 10:45:29 +02006874 // Trigger cpu adapt up, expect no change because quality is most limited.
6875 {
6876 auto previous_sink_wants = source.sink_wants();
6877 // Store current sink wants since we expect no change ind if there is no
6878 // change then last__wants() is not updated.
6879 video_stream_encoder_->TriggerCpuUnderuse();
6880 timestamp_ms += kFrameIntervalMs;
6881 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6882 WaitForEncodedFrame(timestamp_ms);
6883 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6884 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6885 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6886 }
6887
6888 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6889 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006890 timestamp_ms += kFrameIntervalMs;
6891 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006892 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006893 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006894 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6895 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6896 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006897 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6898 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6899 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006900
Evan Shrubsole64469032020-06-11 10:45:29 +02006901 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006902 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006903 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006904 timestamp_ms += kFrameIntervalMs;
6905 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006906 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006907 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006908 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6909 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6910 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6911 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6912 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006913 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006914
6915 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006916 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006917 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006918 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006919 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006920
mflodmancc3d4422017-08-03 08:27:51 -07006921 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006922}
6923
mflodmancc3d4422017-08-03 08:27:51 -07006924TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006925 const int kFrameWidth = 1920;
6926 const int kFrameHeight = 1080;
Ying Wangdb0d5862022-04-28 19:26:22 +02006927 // 2/3 of 1920.
6928 const int kAdaptedFrameWidth = 1280;
6929 // 2/3 of 1080.
6930 const int kAdaptedFrameHeight = 720;
ilnik6b826ef2017-06-16 06:53:48 -07006931 const int kFramerate = 24;
6932
Henrik Boström381d1092020-05-12 18:49:07 +02006933 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006934 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006935 // Trigger reconfigure encoder (without resetting the entire instance).
6936 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006937 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6938 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02006939 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 06:53:48 -07006940 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006941 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006942 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006943 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006944 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006945
6946 video_source_.set_adaptation_enabled(true);
6947
6948 video_source_.IncomingCapturedFrame(
6949 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006950 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006951
6952 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006953 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006954 video_source_.IncomingCapturedFrame(
6955 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006956 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006957
mflodmancc3d4422017-08-03 08:27:51 -07006958 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006959}
6960
mflodmancc3d4422017-08-03 08:27:51 -07006961TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006962 const int kFrameWidth = 1280;
6963 const int kFrameHeight = 720;
6964 const int kLowFps = 2;
6965 const int kHighFps = 30;
6966
Henrik Boström381d1092020-05-12 18:49:07 +02006967 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006968 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006969
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006970 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006971 max_framerate_ = kLowFps;
6972
6973 // Insert 2 seconds of 2fps video.
6974 for (int i = 0; i < kLowFps * 2; ++i) {
6975 video_source_.IncomingCapturedFrame(
6976 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6977 WaitForEncodedFrame(timestamp_ms);
6978 timestamp_ms += 1000 / kLowFps;
6979 }
6980
6981 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006982 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006983 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006984 video_source_.IncomingCapturedFrame(
6985 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6986 WaitForEncodedFrame(timestamp_ms);
6987 timestamp_ms += 1000 / kLowFps;
6988
6989 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6990
6991 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006992 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
Markus Handell2cfc1af2022-08-19 08:16:48 +00006993 constexpr TimeDelta kFrameInterval = TimeDelta::Seconds(1) / kHighFps;
sprang4847ae62017-06-27 07:06:52 -07006994 max_framerate_ = kHighFps;
6995 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6996 video_source_.IncomingCapturedFrame(
6997 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6998 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6999 // be dropped if the encoder hans't been updated with the new higher target
7000 // framerate yet, causing it to overshoot the target bitrate and then
7001 // suffering the wrath of the media optimizer.
Markus Handell2cfc1af2022-08-19 08:16:48 +00007002 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameInterval);
7003 timestamp_ms += kFrameInterval.ms();
sprang4847ae62017-06-27 07:06:52 -07007004 }
7005
7006 // Don expect correct measurement just yet, but it should be higher than
7007 // before.
7008 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
7009
mflodmancc3d4422017-08-03 08:27:51 -07007010 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007011}
7012
mflodmancc3d4422017-08-03 08:27:51 -07007013TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07007014 const int kFrameWidth = 1280;
7015 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 15:09:05 +02007016 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01007017 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02007018 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07007019
Henrik Boström381d1092020-05-12 18:49:07 +02007020 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007021 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07007022 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07007023
7024 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007025 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07007026 video_source_.IncomingCapturedFrame(
7027 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7028 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02007029 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007030
7031 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02007032 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007033 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07007034
7035 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02007036 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007037 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07007038
Per Kjellanderdcef6412020-10-07 15:09:05 +02007039 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07007040 video_source_.IncomingCapturedFrame(
7041 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7042 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02007043 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007044
mflodmancc3d4422017-08-03 08:27:51 -07007045 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007046}
ilnik6b826ef2017-06-16 06:53:48 -07007047
Niels Möller4db138e2018-04-19 09:04:13 +02007048TEST_F(VideoStreamEncoderTest,
7049 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
7050 const int kFrameWidth = 1280;
7051 const int kFrameHeight = 720;
Markus Handell8e4197b2022-05-30 15:45:28 +02007052 const test::ScopedKeyValueConfig kFieldTrials;
7053 const CpuOveruseOptions default_options(kFieldTrials);
Henrik Boström381d1092020-05-12 18:49:07 +02007054 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007055 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007056 video_source_.IncomingCapturedFrame(
7057 CreateFrame(1, kFrameWidth, kFrameHeight));
7058 WaitForEncodedFrame(1);
7059 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7060 .low_encode_usage_threshold_percent,
7061 default_options.low_encode_usage_threshold_percent);
7062 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7063 .high_encode_usage_threshold_percent,
7064 default_options.high_encode_usage_threshold_percent);
7065 video_stream_encoder_->Stop();
7066}
7067
7068TEST_F(VideoStreamEncoderTest,
7069 HigherCpuAdaptationThresholdsForHardwareEncoder) {
7070 const int kFrameWidth = 1280;
7071 const int kFrameHeight = 720;
Markus Handell8e4197b2022-05-30 15:45:28 +02007072 const test::ScopedKeyValueConfig kFieldTrials;
7073 CpuOveruseOptions hardware_options(kFieldTrials);
Niels Möller4db138e2018-04-19 09:04:13 +02007074 hardware_options.low_encode_usage_threshold_percent = 150;
7075 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01007076 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02007077
Henrik Boström381d1092020-05-12 18:49:07 +02007078 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007079 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007080 video_source_.IncomingCapturedFrame(
7081 CreateFrame(1, kFrameWidth, kFrameHeight));
7082 WaitForEncodedFrame(1);
7083 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7084 .low_encode_usage_threshold_percent,
7085 hardware_options.low_encode_usage_threshold_percent);
7086 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7087 .high_encode_usage_threshold_percent,
7088 hardware_options.high_encode_usage_threshold_percent);
7089 video_stream_encoder_->Stop();
7090}
7091
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007092TEST_F(VideoStreamEncoderTest,
7093 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
7094 const int kFrameWidth = 1280;
7095 const int kFrameHeight = 720;
7096
Markus Handell8e4197b2022-05-30 15:45:28 +02007097 const test::ScopedKeyValueConfig kFieldTrials;
7098 const CpuOveruseOptions default_options(kFieldTrials);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007099 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007100 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007101 video_source_.IncomingCapturedFrame(
7102 CreateFrame(1, kFrameWidth, kFrameHeight));
7103 WaitForEncodedFrame(1);
7104 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7105 .low_encode_usage_threshold_percent,
7106 default_options.low_encode_usage_threshold_percent);
7107 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7108 .high_encode_usage_threshold_percent,
7109 default_options.high_encode_usage_threshold_percent);
7110
Markus Handell8e4197b2022-05-30 15:45:28 +02007111 CpuOveruseOptions hardware_options(kFieldTrials);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007112 hardware_options.low_encode_usage_threshold_percent = 150;
7113 hardware_options.high_encode_usage_threshold_percent = 200;
7114 fake_encoder_.SetIsHardwareAccelerated(true);
7115
7116 video_source_.IncomingCapturedFrame(
7117 CreateFrame(2, kFrameWidth, kFrameHeight));
7118 WaitForEncodedFrame(2);
7119
7120 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7121 .low_encode_usage_threshold_percent,
7122 hardware_options.low_encode_usage_threshold_percent);
7123 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7124 .high_encode_usage_threshold_percent,
7125 hardware_options.high_encode_usage_threshold_percent);
7126
7127 video_stream_encoder_->Stop();
7128}
7129
Niels Möller6bb5ab92019-01-11 11:11:10 +01007130TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7131 const int kFrameWidth = 320;
7132 const int kFrameHeight = 240;
7133 const int kFps = 30;
Asa Persson606d3cb2021-10-04 10:07:11 +02007134 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007135 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7136
Henrik Boström381d1092020-05-12 18:49:07 +02007137 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007138 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007139
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007140 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007141 max_framerate_ = kFps;
7142
7143 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7144 fake_encoder_.SimulateOvershoot(1.0);
7145 int num_dropped = 0;
7146 for (int i = 0; i < kNumFramesInRun; ++i) {
7147 video_source_.IncomingCapturedFrame(
7148 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7149 // Wait up to two frame durations for a frame to arrive.
Markus Handell2cfc1af2022-08-19 08:16:48 +00007150 if (!TimedWaitForEncodedFrame(timestamp_ms,
7151 2 * TimeDelta::Seconds(1) / kFps)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01007152 ++num_dropped;
7153 }
7154 timestamp_ms += 1000 / kFps;
7155 }
7156
Erik Språnga8d48ab2019-02-08 14:17:40 +01007157 // Framerate should be measured to be near the expected target rate.
7158 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7159
7160 // Frame drops should be within 5% of expected 0%.
7161 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007162
7163 // Make encoder produce frames at double the expected bitrate during 3 seconds
7164 // of video, verify number of drops. Rate needs to be slightly changed in
7165 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01007166 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02007167 const RateControlSettings trials =
7168 RateControlSettings::ParseFromFieldTrials();
7169 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01007170 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02007171 // frame dropping since the adjuter will try to just lower the target
7172 // bitrate rather than drop frames. If network headroom can be used, it
7173 // doesn't push back as hard so we don't need quite as much overshoot.
7174 // These numbers are unfortunately a bit magical but there's not trivial
7175 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01007176 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01007177 }
7178 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02007179 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007180 kTargetBitrate + DataRate::KilobitsPerSec(1),
7181 kTargetBitrate + DataRate::KilobitsPerSec(1),
7182 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007183 num_dropped = 0;
7184 for (int i = 0; i < kNumFramesInRun; ++i) {
7185 video_source_.IncomingCapturedFrame(
7186 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7187 // Wait up to two frame durations for a frame to arrive.
Markus Handell2cfc1af2022-08-19 08:16:48 +00007188 if (!TimedWaitForEncodedFrame(timestamp_ms,
7189 2 * TimeDelta::Seconds(1) / kFps)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01007190 ++num_dropped;
7191 }
7192 timestamp_ms += 1000 / kFps;
7193 }
7194
Henrik Boström381d1092020-05-12 18:49:07 +02007195 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007196 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007197
7198 // Target framerate should be still be near the expected target, despite
7199 // the frame drops.
7200 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7201
7202 // Frame drops should be within 5% of expected 50%.
7203 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007204
7205 video_stream_encoder_->Stop();
7206}
7207
7208TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7209 const int kFrameWidth = 320;
7210 const int kFrameHeight = 240;
7211 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 10:07:11 +02007212 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007213
7214 ASSERT_GT(max_framerate_, kActualInputFps);
7215
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007216 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007217 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007218 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007219 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007220
7221 // Insert 3 seconds of video, with an input fps lower than configured max.
7222 for (int i = 0; i < kActualInputFps * 3; ++i) {
7223 video_source_.IncomingCapturedFrame(
7224 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7225 // Wait up to two frame durations for a frame to arrive.
7226 WaitForEncodedFrame(timestamp_ms);
7227 timestamp_ms += 1000 / kActualInputFps;
7228 }
7229
7230 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7231
7232 video_stream_encoder_->Stop();
7233}
7234
Markus Handell9a478b52021-11-18 16:07:01 +01007235TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007236 VideoFrame::UpdateRect rect;
Markus Handell9a478b52021-11-18 16:07:01 +01007237 test::FrameForwarder source;
7238 video_stream_encoder_->SetSource(&source,
7239 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02007240 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007241 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007242
Markus Handell9a478b52021-11-18 16:07:01 +01007243 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(1, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007244 WaitForEncodedFrame(1);
7245 // On the very first frame full update should be forced.
7246 rect = fake_encoder_.GetLastUpdateRect();
7247 EXPECT_EQ(rect.offset_x, 0);
7248 EXPECT_EQ(rect.offset_y, 0);
7249 EXPECT_EQ(rect.height, codec_height_);
7250 EXPECT_EQ(rect.width, codec_width_);
Markus Handell9a478b52021-11-18 16:07:01 +01007251 // Frame with NTP timestamp 2 will be dropped due to outstanding frames
7252 // scheduled for processing during encoder queue processing of frame 2.
7253 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(2, nullptr, 1));
7254 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(3, nullptr, 10));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007255 WaitForEncodedFrame(3);
7256 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7257 rect = fake_encoder_.GetLastUpdateRect();
7258 EXPECT_EQ(rect.offset_x, 1);
7259 EXPECT_EQ(rect.offset_y, 0);
7260 EXPECT_EQ(rect.width, 10);
7261 EXPECT_EQ(rect.height, 1);
7262
Markus Handell9a478b52021-11-18 16:07:01 +01007263 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(4, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007264 WaitForEncodedFrame(4);
7265 // Previous frame was encoded, so no accumulation should happen.
7266 rect = fake_encoder_.GetLastUpdateRect();
7267 EXPECT_EQ(rect.offset_x, 0);
7268 EXPECT_EQ(rect.offset_y, 0);
7269 EXPECT_EQ(rect.width, 1);
7270 EXPECT_EQ(rect.height, 1);
7271
7272 video_stream_encoder_->Stop();
7273}
7274
Erik Språngd7329ca2019-02-21 21:19:53 +01007275TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007276 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007277 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007278
7279 // First frame is always keyframe.
7280 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7281 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007282 EXPECT_THAT(
7283 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007284 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007285
7286 // Insert delta frame.
7287 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7288 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007289 EXPECT_THAT(
7290 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007291 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007292
7293 // Request next frame be a key-frame.
7294 video_stream_encoder_->SendKeyFrame();
7295 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7296 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007297 EXPECT_THAT(
7298 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007299 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007300
7301 video_stream_encoder_->Stop();
7302}
7303
7304TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7305 // Setup simulcast with three streams.
7306 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007307 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007308 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7309 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007310 // Wait for all three layers before triggering event.
7311 sink_.SetNumExpectedLayers(3);
7312
7313 // First frame is always keyframe.
7314 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7315 WaitForEncodedFrame(1);
7316 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007317 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7318 VideoFrameType::kVideoFrameKey,
7319 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007320
7321 // Insert delta frame.
7322 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7323 WaitForEncodedFrame(2);
7324 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007325 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7326 VideoFrameType::kVideoFrameDelta,
7327 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007328
7329 // Request next frame be a key-frame.
7330 // Only first stream is configured to produce key-frame.
7331 video_stream_encoder_->SendKeyFrame();
7332 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7333 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007334
7335 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7336 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007337 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007338 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007339 VideoFrameType::kVideoFrameKey,
7340 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007341
7342 video_stream_encoder_->Stop();
7343}
7344
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007345TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007346 // SPS contains VUI with restrictions on the maximum number of reordered
7347 // pictures, there is no need to rewrite the bitstream to enable faster
7348 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007349 ResetEncoder("H264", 1, 1, 1, false);
7350
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007351 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007352 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007353 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007354
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007355 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 10:07:11 +02007356 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007357
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007358 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7359 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007360
7361 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007362 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007363
7364 video_stream_encoder_->Stop();
7365}
7366
7367TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007368 // SPS does not contain VUI, the bitstream is will be rewritten with added
7369 // VUI with restrictions on the maximum number of reordered pictures to
7370 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007371 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7372 0x00, 0x00, 0x03, 0x03, 0xF4,
7373 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007374 ResetEncoder("H264", 1, 1, 1, false);
7375
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007376 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007377 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007378 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007379
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007380 fake_encoder_.SetEncodedImageData(
7381 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007382
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007383 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7384 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007385
7386 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007387 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007388
7389 video_stream_encoder_->Stop();
7390}
7391
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007392TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7393 const int kFrameWidth = 1280;
7394 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 10:07:11 +02007395 const DataRate kTargetBitrate =
7396 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007397
Henrik Boström381d1092020-05-12 18:49:07 +02007398 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007399 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007400 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7401
7402 // Insert a first video frame. It should be dropped because of downscale in
7403 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007404 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007405 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7406 frame.set_rotation(kVideoRotation_270);
7407 video_source_.IncomingCapturedFrame(frame);
7408
7409 ExpectDroppedFrame();
7410
7411 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007412 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007413 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7414 frame.set_rotation(kVideoRotation_90);
7415 video_source_.IncomingCapturedFrame(frame);
7416
7417 WaitForEncodedFrame(timestamp_ms);
7418 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7419
7420 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007421 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007422 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7423 frame.set_rotation(kVideoRotation_180);
7424 video_source_.IncomingCapturedFrame(frame);
7425
7426 WaitForEncodedFrame(timestamp_ms);
7427 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7428
7429 video_stream_encoder_->Stop();
7430}
7431
Erik Språng5056af02019-09-02 15:53:11 +02007432TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7433 const int kFrameWidth = 320;
7434 const int kFrameHeight = 180;
7435
7436 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007437 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007438 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7439 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7440 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007441 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007442 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007443 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007444
7445 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007446 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007447 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7448 frame.set_rotation(kVideoRotation_270);
7449 video_source_.IncomingCapturedFrame(frame);
7450 WaitForEncodedFrame(timestamp_ms);
7451
7452 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 10:07:11 +02007453 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007454 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7455 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007456 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007457 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007458 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007459 /*link_allocation=*/target_rate,
7460 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007461 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007462 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007463 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7464
7465 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7466 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7467 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007468 DataRate allocation_sum =
7469 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007470 EXPECT_EQ(min_rate, allocation_sum);
7471 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7472
7473 video_stream_encoder_->Stop();
7474}
7475
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007476TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007477 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007478 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007479 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007480 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007481 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7482 WaitForEncodedFrame(1);
7483
7484 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7485 ASSERT_TRUE(prev_rate_settings.has_value());
7486 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7487 kDefaultFramerate);
7488
7489 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7490 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7491 timestamp_ms += 1000 / kDefaultFramerate;
7492 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7493 WaitForEncodedFrame(timestamp_ms);
7494 }
7495 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7496 kDefaultFramerate);
7497 // Capture larger frame to trigger a reconfigure.
7498 codec_height_ *= 2;
7499 codec_width_ *= 2;
7500 timestamp_ms += 1000 / kDefaultFramerate;
7501 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7502 WaitForEncodedFrame(timestamp_ms);
7503
7504 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7505 auto current_rate_settings =
7506 fake_encoder_.GetAndResetLastRateControlSettings();
7507 // Ensure we have actually reconfigured twice
7508 // The rate settings should have been set again even though
7509 // they haven't changed.
7510 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007511 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007512
7513 video_stream_encoder_->Stop();
7514}
7515
philipeld9cc8c02019-09-16 14:53:40 +02007516struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007517 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007518 MOCK_METHOD(void,
7519 RequestEncoderSwitch,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007520 (const webrtc::SdpVideoFormat& format,
7521 bool allow_default_fallback),
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007522 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007523};
7524
philipel9b058032020-02-10 11:30:00 +01007525TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7526 constexpr int kDontCare = 100;
7527 StrictMock<MockEncoderSelector> encoder_selector;
7528 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7529 &fake_encoder_, &encoder_selector);
7530 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7531
7532 // Reset encoder for new configuration to take effect.
7533 ConfigureEncoder(video_encoder_config_.Copy());
7534
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007535 EXPECT_CALL(encoder_selector, OnCurrentEncoder);
philipel9b058032020-02-10 11:30:00 +01007536
7537 video_source_.IncomingCapturedFrame(
7538 CreateFrame(kDontCare, kDontCare, kDontCare));
Markus Handell28c71802021-11-08 10:11:55 +01007539 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007540 video_stream_encoder_->Stop();
7541
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007542 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007543 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007544 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7545 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007546 video_stream_encoder_.reset();
7547}
7548
7549TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7550 constexpr int kDontCare = 100;
7551
7552 NiceMock<MockEncoderSelector> encoder_selector;
7553 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7554 video_send_config_.encoder_settings.encoder_switch_request_callback =
7555 &switch_callback;
7556 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7557 &fake_encoder_, &encoder_selector);
7558 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7559
7560 // Reset encoder for new configuration to take effect.
7561 ConfigureEncoder(video_encoder_config_.Copy());
7562
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007563 ON_CALL(encoder_selector, OnAvailableBitrate)
philipel9b058032020-02-10 11:30:00 +01007564 .WillByDefault(Return(SdpVideoFormat("AV1")));
7565 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007566 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7567 /*allow_default_fallback=*/false));
philipel9b058032020-02-10 11:30:00 +01007568
Henrik Boström381d1092020-05-12 18:49:07 +02007569 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007570 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7571 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7572 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007573 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007574 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007575 /*cwnd_reduce_ratio=*/0);
Markus Handell28c71802021-11-08 10:11:55 +01007576 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007577
7578 video_stream_encoder_->Stop();
7579}
7580
philipel6daa3042022-04-11 10:48:28 +02007581TEST_F(VideoStreamEncoderTest, EncoderSelectorResolutionSwitch) {
7582 NiceMock<MockEncoderSelector> encoder_selector;
7583 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7584 video_send_config_.encoder_settings.encoder_switch_request_callback =
7585 &switch_callback;
7586 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7587 &fake_encoder_, &encoder_selector);
7588 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7589
7590 // Reset encoder for new configuration to take effect.
7591 ConfigureEncoder(video_encoder_config_.Copy());
7592
7593 EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(640, 480)))
7594 .WillOnce(Return(absl::nullopt));
7595 EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(320, 240)))
7596 .WillOnce(Return(SdpVideoFormat("AV1")));
7597 EXPECT_CALL(switch_callback,
7598 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7599 /*allow_default_fallback=*/false));
7600
7601 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7602 /*target_bitrate=*/DataRate::KilobitsPerSec(800),
7603 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(1000),
7604 /*link_allocation=*/DataRate::KilobitsPerSec(1000),
7605 /*fraction_lost=*/0,
7606 /*round_trip_time_ms=*/0,
7607 /*cwnd_reduce_ratio=*/0);
7608
7609 video_source_.IncomingCapturedFrame(CreateFrame(1, 640, 480));
7610 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 480));
7611 video_source_.IncomingCapturedFrame(CreateFrame(3, 320, 240));
7612
7613 AdvanceTime(TimeDelta::Zero());
7614
7615 video_stream_encoder_->Stop();
7616}
7617
philipel9b058032020-02-10 11:30:00 +01007618TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7619 constexpr int kSufficientBitrateToNotDrop = 1000;
7620 constexpr int kDontCare = 100;
7621
7622 NiceMock<MockVideoEncoder> video_encoder;
7623 NiceMock<MockEncoderSelector> encoder_selector;
7624 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7625 video_send_config_.encoder_settings.encoder_switch_request_callback =
7626 &switch_callback;
7627 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7628 &video_encoder, &encoder_selector);
7629 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7630
7631 // Reset encoder for new configuration to take effect.
7632 ConfigureEncoder(video_encoder_config_.Copy());
7633
7634 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7635 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7636 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007637 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007638 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7639 /*stable_target_bitrate=*/
7640 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7641 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007642 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007643 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007644 /*cwnd_reduce_ratio=*/0);
7645
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007646 ON_CALL(video_encoder, Encode)
philipel9b058032020-02-10 11:30:00 +01007647 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007648 ON_CALL(encoder_selector, OnEncoderBroken)
philipel9b058032020-02-10 11:30:00 +01007649 .WillByDefault(Return(SdpVideoFormat("AV2")));
7650
7651 rtc::Event encode_attempted;
7652 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007653 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7654 /*allow_default_fallback=*/true))
7655 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
philipel9b058032020-02-10 11:30:00 +01007656
7657 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
Markus Handell2cfc1af2022-08-19 08:16:48 +00007658 encode_attempted.Wait(TimeDelta::Seconds(3));
philipel9b058032020-02-10 11:30:00 +01007659
Markus Handell28c71802021-11-08 10:11:55 +01007660 AdvanceTime(TimeDelta::Zero());
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007661
philipel9b058032020-02-10 11:30:00 +01007662 video_stream_encoder_->Stop();
7663
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007664 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7665 // to it's factory, so in order for the encoder instance in the
7666 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7667 // reset the `video_stream_encoder_` here.
7668 video_stream_encoder_.reset();
7669}
7670
7671TEST_F(VideoStreamEncoderTest, SwitchEncoderOnInitFailureWithEncoderSelector) {
7672 NiceMock<MockVideoEncoder> video_encoder;
7673 NiceMock<MockEncoderSelector> encoder_selector;
7674 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7675 video_send_config_.encoder_settings.encoder_switch_request_callback =
7676 &switch_callback;
7677 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7678 &video_encoder, &encoder_selector);
7679 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7680
7681 // Reset encoder for new configuration to take effect.
7682 ConfigureEncoder(video_encoder_config_.Copy());
7683
7684 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7685 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7686 /*round_trip_time_ms=*/0,
7687 /*cwnd_reduce_ratio=*/0);
7688 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7689
7690 ON_CALL(video_encoder, InitEncode(_, _))
7691 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7692 ON_CALL(encoder_selector, OnEncoderBroken)
7693 .WillByDefault(Return(SdpVideoFormat("AV2")));
7694
7695 rtc::Event encode_attempted;
7696 EXPECT_CALL(switch_callback,
7697 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7698 /*allow_default_fallback=*/true))
7699 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7700
7701 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
Markus Handell2cfc1af2022-08-19 08:16:48 +00007702 encode_attempted.Wait(TimeDelta::Seconds(3));
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007703
7704 AdvanceTime(TimeDelta::Zero());
7705
7706 video_stream_encoder_->Stop();
7707
7708 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7709 // to it's factory, so in order for the encoder instance in the
7710 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7711 // reset the `video_stream_encoder_` here.
7712 video_stream_encoder_.reset();
7713}
7714
7715TEST_F(VideoStreamEncoderTest,
7716 SwitchEncoderOnInitFailureWithoutEncoderSelector) {
7717 NiceMock<MockVideoEncoder> video_encoder;
7718 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7719 video_send_config_.encoder_settings.encoder_switch_request_callback =
7720 &switch_callback;
7721 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7722 &video_encoder, /*encoder_selector=*/nullptr);
7723 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7724
7725 // Reset encoder for new configuration to take effect.
7726 ConfigureEncoder(video_encoder_config_.Copy());
7727
7728 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7729 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7730 /*round_trip_time_ms=*/0,
7731 /*cwnd_reduce_ratio=*/0);
7732 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7733
7734 ON_CALL(video_encoder, InitEncode(_, _))
7735 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7736
7737 rtc::Event encode_attempted;
7738 EXPECT_CALL(switch_callback,
7739 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "VP8"),
7740 /*allow_default_fallback=*/true))
7741 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7742
7743 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
Markus Handell2cfc1af2022-08-19 08:16:48 +00007744 encode_attempted.Wait(TimeDelta::Seconds(3));
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007745
7746 AdvanceTime(TimeDelta::Zero());
7747
7748 video_stream_encoder_->Stop();
7749
7750 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007751 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007752 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7753 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007754 video_stream_encoder_.reset();
7755}
7756
Byoungchan Lee13fe3672022-04-06 10:44:42 +09007757TEST_F(VideoStreamEncoderTest, NullEncoderReturnSwitch) {
7758 // As a variant of EncoderSelectorBrokenEncoderSwitch, when a null
7759 // VideoEncoder is passed in encoder_factory, it checks whether
7760 // Codec Switch occurs without a crash.
7761 constexpr int kSufficientBitrateToNotDrop = 1000;
7762 constexpr int kDontCare = 100;
7763
7764 NiceMock<MockEncoderSelector> encoder_selector;
7765 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7766 video_send_config_.encoder_settings.encoder_switch_request_callback =
7767 &switch_callback;
7768 auto encoder_factory =
7769 std::make_unique<test::VideoEncoderNullableProxyFactory>(
7770 /*encoder=*/nullptr, &encoder_selector);
7771 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7772
7773 // Reset encoder for new configuration to take effect.
7774 ConfigureEncoder(video_encoder_config_.Copy());
7775 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7776 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7777 // not fail.
7778 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7779 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7780 /*stable_target_bitrate=*/
7781 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7782 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7783 /*fraction_lost=*/0,
7784 /*round_trip_time_ms=*/0,
7785 /*cwnd_reduce_ratio=*/0);
7786 ON_CALL(encoder_selector, OnEncoderBroken)
7787 .WillByDefault(Return(SdpVideoFormat("AV2")));
7788 rtc::Event encode_attempted;
7789 EXPECT_CALL(switch_callback,
7790 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7791 /*allow_default_fallback=*/_))
7792 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7793
7794 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
Markus Handell2cfc1af2022-08-19 08:16:48 +00007795 encode_attempted.Wait(TimeDelta::Seconds(3));
Byoungchan Lee13fe3672022-04-06 10:44:42 +09007796
7797 AdvanceTime(TimeDelta::Zero());
7798
7799 video_stream_encoder_->Stop();
7800
7801 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7802 // to it's factory, so in order for the encoder instance in the
7803 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7804 // reset the `video_stream_encoder_` here.
7805 video_stream_encoder_.reset();
7806}
7807
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007808TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007809 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007810 const int kFrameWidth = 320;
7811 const int kFrameHeight = 180;
7812
7813 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007814 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007815 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007816 /*target_bitrate=*/rate,
7817 /*stable_target_bitrate=*/rate,
7818 /*link_allocation=*/rate,
7819 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007820 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007821 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007822
7823 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007824 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007825 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7826 frame.set_rotation(kVideoRotation_270);
7827 video_source_.IncomingCapturedFrame(frame);
7828 WaitForEncodedFrame(timestamp_ms);
7829 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7830
7831 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007832 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007833 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007834 /*target_bitrate=*/new_stable_rate,
7835 /*stable_target_bitrate=*/new_stable_rate,
7836 /*link_allocation=*/rate,
7837 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007838 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007839 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007840 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7841 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7842 video_stream_encoder_->Stop();
7843}
7844
7845TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007846 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007847 const int kFrameWidth = 320;
7848 const int kFrameHeight = 180;
7849
7850 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007851 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007852 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007853 /*target_bitrate=*/rate,
7854 /*stable_target_bitrate=*/rate,
7855 /*link_allocation=*/rate,
7856 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007857 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007858 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007859
7860 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007861 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007862 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7863 frame.set_rotation(kVideoRotation_270);
7864 video_source_.IncomingCapturedFrame(frame);
7865 WaitForEncodedFrame(timestamp_ms);
7866 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7867
7868 // Set a higher target rate without changing the link_allocation. Should not
7869 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007870 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007871 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007872 /*target_bitrate=*/rate,
7873 /*stable_target_bitrate=*/new_stable_rate,
7874 /*link_allocation=*/rate,
7875 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007876 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007877 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007878 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7879 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7880 video_stream_encoder_->Stop();
7881}
7882
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007883TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01007884 test::ScopedKeyValueConfig field_trials(
7885 field_trials_,
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007886 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7887 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7888 const int kFramerateFps = 30;
7889 const int kWidth = 1920;
7890 const int kHeight = 1080;
7891 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7892 // Works on screenshare mode.
7893 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7894 // We rely on the automatic resolution adaptation, but we handle framerate
7895 // adaptation manually by mocking the stats proxy.
7896 video_source_.set_adaptation_enabled(true);
7897
7898 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007899 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007900 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007901 video_stream_encoder_->SetSource(&video_source_,
7902 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007903 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007904
7905 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7906 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7907
7908 // Pass enough frames with the full update to trigger animation detection.
7909 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007910 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007911 frame.set_ntp_time_ms(timestamp_ms);
7912 frame.set_timestamp_us(timestamp_ms * 1000);
7913 video_source_.IncomingCapturedFrame(frame);
7914 WaitForEncodedFrame(timestamp_ms);
7915 }
7916
7917 // Resolution should be limited.
7918 rtc::VideoSinkWants expected;
7919 expected.max_framerate_fps = kFramerateFps;
7920 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007921 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007922
7923 // Pass one frame with no known update.
7924 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007925 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007926 frame.set_ntp_time_ms(timestamp_ms);
7927 frame.set_timestamp_us(timestamp_ms * 1000);
7928 frame.clear_update_rect();
7929
7930 video_source_.IncomingCapturedFrame(frame);
7931 WaitForEncodedFrame(timestamp_ms);
7932
7933 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007934 EXPECT_THAT(video_source_.sink_wants(),
7935 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007936
7937 video_stream_encoder_->Stop();
7938}
7939
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007940TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7941 const int kWidth = 720; // 540p adapted down.
7942 const int kHeight = 405;
7943 const int kNumFrames = 3;
7944 // Works on screenshare mode.
7945 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7946 /*num_spatial_layers=*/2, /*screenshare=*/true);
7947
7948 video_source_.set_adaptation_enabled(true);
7949
7950 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007951 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007952
7953 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7954
7955 // Pass enough frames with the full update to trigger animation detection.
7956 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007957 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007958 frame.set_ntp_time_ms(timestamp_ms);
7959 frame.set_timestamp_us(timestamp_ms * 1000);
7960 video_source_.IncomingCapturedFrame(frame);
7961 WaitForEncodedFrame(timestamp_ms);
7962 }
7963
7964 video_stream_encoder_->Stop();
7965}
7966
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007967TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7968 const float downscale_factors[] = {4.0, 2.0, 1.0};
7969 const int number_layers =
7970 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7971 VideoEncoderConfig config;
7972 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7973 for (int i = 0; i < number_layers; ++i) {
7974 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7975 config.simulcast_layers[i].active = true;
7976 }
7977 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007978 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007979 "VP8", /*max qp*/ 56, /*screencast*/ false,
7980 /*screenshare enabled*/ false);
7981 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007982 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7983 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007984
7985 // First initialization.
7986 // Encoder should be initialized. Next frame should be key frame.
7987 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7988 sink_.SetNumExpectedLayers(number_layers);
7989 int64_t timestamp_ms = kFrameIntervalMs;
7990 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7991 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007992 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007993 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7994 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7995 VideoFrameType::kVideoFrameKey,
7996 VideoFrameType::kVideoFrameKey}));
7997
7998 // Disable top layer.
7999 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
8000 config.simulcast_layers[number_layers - 1].active = false;
8001 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8002 sink_.SetNumExpectedLayers(number_layers - 1);
8003 timestamp_ms += kFrameIntervalMs;
8004 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8005 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008006 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008007 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8008 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
8009 VideoFrameType::kVideoFrameDelta,
8010 VideoFrameType::kVideoFrameDelta}));
8011
8012 // Re-enable top layer.
8013 // Encoder should be re-initialized. Next frame should be key frame.
8014 config.simulcast_layers[number_layers - 1].active = true;
8015 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8016 sink_.SetNumExpectedLayers(number_layers);
8017 timestamp_ms += kFrameIntervalMs;
8018 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8019 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008020 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008021 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8022 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
8023 VideoFrameType::kVideoFrameKey,
8024 VideoFrameType::kVideoFrameKey}));
8025
8026 // Top layer max rate change.
8027 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
8028 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
8029 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8030 sink_.SetNumExpectedLayers(number_layers);
8031 timestamp_ms += kFrameIntervalMs;
8032 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8033 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008034 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008035 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8036 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
8037 VideoFrameType::kVideoFrameDelta,
8038 VideoFrameType::kVideoFrameDelta}));
8039
8040 // Top layer resolution change.
8041 // Encoder should be re-initialized. Next frame should be key frame.
8042 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
8043 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8044 sink_.SetNumExpectedLayers(number_layers);
8045 timestamp_ms += kFrameIntervalMs;
8046 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8047 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008048 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008049 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8050 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
8051 VideoFrameType::kVideoFrameKey,
8052 VideoFrameType::kVideoFrameKey}));
8053 video_stream_encoder_->Stop();
8054}
8055
Henrik Boström1124ed12021-02-25 10:30:39 +01008056TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
8057 const int kFrameWidth = 1280;
8058 const int kFrameHeight = 720;
8059
8060 SetUp();
8061 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008062 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01008063
8064 // Capturing a frame should reconfigure the encoder and expose the encoder
8065 // resolution, which is the same as the input frame.
8066 int64_t timestamp_ms = kFrameIntervalMs;
8067 video_source_.IncomingCapturedFrame(
8068 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8069 WaitForEncodedFrame(timestamp_ms);
8070 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8071 EXPECT_THAT(video_source_.sink_wants().resolutions,
8072 ::testing::ElementsAreArray(
8073 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
8074
8075 video_stream_encoder_->Stop();
8076}
8077
8078TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
8079 // Pick downscale factors such that we never encode at full resolution - this
8080 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 16:22:11 +02008081 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 10:30:39 +01008082 // encoder should not ask for the frame resolution. This allows video frames
8083 // to have the appearence of one resolution but optimize its internal buffers
8084 // for what is actually encoded.
8085 const size_t kNumSimulcastLayers = 3u;
8086 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
8087 const int kFrameWidth = 1280;
8088 const int kFrameHeight = 720;
8089 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8090 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8091 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8092 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8093 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8094 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8095
8096 VideoEncoderConfig config;
8097 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
8098 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
8099 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
8100 config.simulcast_layers[i].active = true;
8101 }
8102 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02008103 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01008104 "VP8", /*max qp*/ 56, /*screencast*/ false,
8105 /*screenshare enabled*/ false);
8106 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008107 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8108 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01008109
8110 // Capture a frame with all layers active.
8111 int64_t timestamp_ms = kFrameIntervalMs;
8112 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
8113 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8114 video_source_.IncomingCapturedFrame(
8115 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8116 WaitForEncodedFrame(timestamp_ms);
8117 // Expect encoded resolutions to match the expected simulcast layers.
8118 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8119 EXPECT_THAT(
8120 video_source_.sink_wants().resolutions,
8121 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
8122
8123 // Capture a frame with one of the layers inactive.
8124 timestamp_ms += kFrameIntervalMs;
8125 config.simulcast_layers[2].active = false;
8126 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
8127 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8128 video_source_.IncomingCapturedFrame(
8129 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8130 WaitForEncodedFrame(timestamp_ms);
8131
8132 // Expect encoded resolutions to match the expected simulcast layers.
8133 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8134 EXPECT_THAT(video_source_.sink_wants().resolutions,
8135 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
8136
8137 // Capture a frame with all but one layer turned off.
8138 timestamp_ms += kFrameIntervalMs;
8139 config.simulcast_layers[1].active = false;
8140 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
8141 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8142 video_source_.IncomingCapturedFrame(
8143 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8144 WaitForEncodedFrame(timestamp_ms);
8145
8146 // Expect encoded resolutions to match the expected simulcast layers.
8147 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8148 EXPECT_THAT(video_source_.sink_wants().resolutions,
8149 ::testing::ElementsAreArray({kLayer0Size}));
8150
8151 video_stream_encoder_->Stop();
8152}
8153
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008154TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008155 ResetEncoder("VP8", 1, 1, 1, false);
8156
Niels Möller8b692902021-06-14 12:04:57 +02008157 // Force encoder reconfig.
8158 video_source_.IncomingCapturedFrame(
8159 CreateFrame(1, codec_width_, codec_height_));
8160 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8161
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008162 // Set QP on encoded frame and pass the frame to encode complete callback.
8163 // Since QP is present QP parsing won't be triggered and the original value
8164 // should be kept.
8165 EncodedImage encoded_image;
8166 encoded_image.qp_ = 123;
8167 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8168 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8169 CodecSpecificInfo codec_info;
8170 codec_info.codecType = kVideoCodecVP8;
8171 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
Markus Handell2cfc1af2022-08-19 08:16:48 +00008172 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeout));
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008173 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
8174 video_stream_encoder_->Stop();
8175}
8176
8177TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008178 ResetEncoder("VP8", 1, 1, 1, false);
8179
Niels Möller8b692902021-06-14 12:04:57 +02008180 // Force encoder reconfig.
8181 video_source_.IncomingCapturedFrame(
8182 CreateFrame(1, codec_width_, codec_height_));
8183 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8184
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008185 // Pass an encoded frame without QP to encode complete callback. QP should be
8186 // parsed and set.
8187 EncodedImage encoded_image;
8188 encoded_image.qp_ = -1;
8189 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8190 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8191 CodecSpecificInfo codec_info;
8192 codec_info.codecType = kVideoCodecVP8;
8193 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
Markus Handell2cfc1af2022-08-19 08:16:48 +00008194 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeout));
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008195 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
8196 video_stream_encoder_->Stop();
8197}
8198
8199TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01008200 webrtc::test::ScopedKeyValueConfig field_trials(
8201 field_trials_, "WebRTC-QpParsingKillSwitch/Enabled/");
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008202
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008203 ResetEncoder("VP8", 1, 1, 1, false);
8204
Niels Möller8b692902021-06-14 12:04:57 +02008205 // Force encoder reconfig.
8206 video_source_.IncomingCapturedFrame(
8207 CreateFrame(1, codec_width_, codec_height_));
8208 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8209
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008210 EncodedImage encoded_image;
8211 encoded_image.qp_ = -1;
8212 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8213 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8214 CodecSpecificInfo codec_info;
8215 codec_info.codecType = kVideoCodecVP8;
8216 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
Markus Handell2cfc1af2022-08-19 08:16:48 +00008217 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeout));
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008218 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
8219 video_stream_encoder_->Stop();
8220}
8221
Sergey Silkind19e3b92021-03-16 10:05:30 +00008222TEST_F(VideoStreamEncoderTest,
8223 QualityScalingNotAllowed_QualityScalingDisabled) {
8224 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8225
8226 // Disable scaling settings in encoder info.
8227 fake_encoder_.SetQualityScaling(false);
8228 // Disable quality scaling in encoder config.
8229 video_encoder_config.is_quality_scaling_allowed = false;
8230 ConfigureEncoder(std::move(video_encoder_config));
8231
8232 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008233 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008234
8235 test::FrameForwarder source;
8236 video_stream_encoder_->SetSource(
8237 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8238 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8239 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8240
8241 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8242 WaitForEncodedFrame(1);
8243 video_stream_encoder_->TriggerQualityLow();
8244 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8245
8246 video_stream_encoder_->Stop();
8247}
8248
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008249TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
8250 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8251
8252 // Disable scaling settings in encoder info.
8253 fake_encoder_.SetQualityScaling(false);
8254 // Set QP trusted in encoder info.
8255 fake_encoder_.SetIsQpTrusted(true);
8256 // Enable quality scaling in encoder config.
8257 video_encoder_config.is_quality_scaling_allowed = false;
8258 ConfigureEncoder(std::move(video_encoder_config));
8259
8260 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008261 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008262
8263 test::FrameForwarder source;
8264 video_stream_encoder_->SetSource(
8265 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8266 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8267 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8268
8269 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8270 WaitForEncodedFrame(1);
8271 video_stream_encoder_->TriggerQualityLow();
8272 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8273
8274 video_stream_encoder_->Stop();
8275}
8276
Shuhai Pengf2707702021-09-29 17:19:44 +08008277TEST_F(VideoStreamEncoderTest,
8278 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8279 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8280
8281 // Disable scaling settings in encoder info.
8282 fake_encoder_.SetQualityScaling(false);
8283 // Set QP trusted in encoder info.
8284 fake_encoder_.SetIsQpTrusted(true);
8285 // Enable quality scaling in encoder config.
8286 video_encoder_config.is_quality_scaling_allowed = false;
8287 ConfigureEncoder(std::move(video_encoder_config));
8288
8289 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008290 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008291
8292 test::FrameForwarder source;
8293 video_stream_encoder_->SetSource(
8294 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8295 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8296 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8297
8298 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8299 WaitForEncodedFrame(1);
8300 video_stream_encoder_->TriggerQualityLow();
8301 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8302
8303 video_stream_encoder_->Stop();
8304}
8305
8306TEST_F(VideoStreamEncoderTest,
8307 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8308 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8309
8310 // Disable scaling settings in encoder info.
8311 fake_encoder_.SetQualityScaling(false);
8312 // Set QP trusted in encoder info.
8313 fake_encoder_.SetIsQpTrusted(false);
8314 // Enable quality scaling in encoder config.
8315 video_encoder_config.is_quality_scaling_allowed = false;
8316 ConfigureEncoder(std::move(video_encoder_config));
8317
8318 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008319 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008320
8321 test::FrameForwarder source;
8322 video_stream_encoder_->SetSource(
8323 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8324 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8325 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8326
8327 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8328 WaitForEncodedFrame(1);
8329 video_stream_encoder_->TriggerQualityLow();
8330 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8331
8332 video_stream_encoder_->Stop();
8333}
8334
8335TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8336 // Set QP trusted in encoder info.
8337 fake_encoder_.SetIsQpTrusted(false);
8338
8339 const int MinEncBitrateKbps = 30;
8340 const int MaxEncBitrateKbps = 100;
8341 const int MinStartBitrateKbp = 50;
8342 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8343 /*frame_size_pixels=*/codec_width_ * codec_height_,
8344 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8345 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8346 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8347
8348 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008349 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008350
8351 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8352
8353 VideoEncoderConfig video_encoder_config;
8354 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8355 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8356 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8357 MinEncBitrateKbps * 1000;
8358 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8359 kMaxPayloadLength);
8360
8361 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8362 WaitForEncodedFrame(1);
8363 EXPECT_EQ(
8364 MaxEncBitrateKbps,
8365 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8366 EXPECT_EQ(
8367 MinEncBitrateKbps,
8368 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8369
8370 video_stream_encoder_->Stop();
8371}
8372
8373TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8374 // Set QP trusted in encoder info.
8375 fake_encoder_.SetIsQpTrusted(false);
8376
8377 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8378 EncoderInfoSettings::
8379 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8380 codec_width_ * codec_height_,
8381 EncoderInfoSettings::
8382 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8383 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8384
8385 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8386 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8387 const int TargetEncBitrate = MaxEncBitrate;
8388 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8389 DataRate::BitsPerSec(TargetEncBitrate),
8390 DataRate::BitsPerSec(TargetEncBitrate),
8391 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8392
8393 VideoEncoderConfig video_encoder_config;
8394 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8395 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8396 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8397 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8398 kMaxPayloadLength);
8399
8400 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8401 WaitForEncodedFrame(1);
8402 EXPECT_EQ(
8403 MaxEncBitrate / 1000,
8404 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8405 EXPECT_EQ(
8406 MinEncBitrate / 1000,
8407 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8408
8409 video_stream_encoder_->Stop();
8410}
8411
Erik Språnge4589cb2022-04-06 16:44:30 +02008412TEST_F(VideoStreamEncoderTest, NormalComplexityWithMoreThanTwoCores) {
8413 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8414 /*num_spatial_layers=*/1,
8415 /*screenshare=*/false, /*allocation_callback_type=*/
8416 VideoStreamEncoder::BitrateAllocationCallbackType::
8417 kVideoBitrateAllocationWhenScreenSharing,
8418 /*num_cores=*/3);
8419
8420 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8421 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8422 video_source_.IncomingCapturedFrame(
8423 CreateFrame(1, /*width=*/320, /*height=*/180));
8424 WaitForEncodedFrame(1);
8425 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8426 VideoCodecComplexity::kComplexityNormal);
8427 video_stream_encoder_->Stop();
8428}
8429
8430TEST_F(VideoStreamEncoderTest,
8431 NormalComplexityWhenLowTierOptimizationsAreDisabled) {
8432 webrtc::test::ScopedKeyValueConfig field_trials(
8433 field_trials_, "WebRTC-VP9-LowTierOptimizations/Disabled/");
8434
8435 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8436 /*num_spatial_layers=*/1,
8437 /*screenshare=*/false, /*allocation_callback_type=*/
8438 VideoStreamEncoder::BitrateAllocationCallbackType::
8439 kVideoBitrateAllocationWhenScreenSharing,
8440 /*num_cores=*/2);
8441
8442 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8443 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8444 video_source_.IncomingCapturedFrame(
8445 CreateFrame(1, /*width=*/320, /*height=*/180));
8446 WaitForEncodedFrame(1);
8447 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8448 VideoCodecComplexity::kComplexityNormal);
8449 video_stream_encoder_->Stop();
8450}
8451
8452TEST_F(VideoStreamEncoderTest, LowComplexityWithTwoCores) {
8453 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8454 /*num_spatial_layers=*/1,
8455 /*screenshare=*/false, /*allocation_callback_type=*/
8456 VideoStreamEncoder::BitrateAllocationCallbackType::
8457 kVideoBitrateAllocationWhenScreenSharing,
8458 /*num_cores=*/2);
8459
8460 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8461 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8462 video_source_.IncomingCapturedFrame(
8463 CreateFrame(1, /*width=*/320, /*height=*/180));
8464 WaitForEncodedFrame(1);
8465 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8466 VideoCodecComplexity::kComplexityLow);
8467 video_stream_encoder_->Stop();
8468}
8469
Sergey Silkind19e3b92021-03-16 10:05:30 +00008470#if !defined(WEBRTC_IOS)
8471// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8472// disabled by default on iOS.
8473TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8474 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8475
8476 // Disable scaling settings in encoder info.
8477 fake_encoder_.SetQualityScaling(false);
8478 // Enable quality scaling in encoder config.
8479 video_encoder_config.is_quality_scaling_allowed = true;
8480 ConfigureEncoder(std::move(video_encoder_config));
8481
8482 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008483 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008484
8485 test::FrameForwarder source;
8486 video_stream_encoder_->SetSource(
8487 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8488 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8489 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8490
8491 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8492 WaitForEncodedFrame(1);
8493 video_stream_encoder_->TriggerQualityLow();
8494 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8495
8496 video_stream_encoder_->Stop();
8497}
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008498
8499TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8500 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8501
8502 // Disable scaling settings in encoder info.
8503 fake_encoder_.SetQualityScaling(false);
8504 // Set QP trusted in encoder info.
8505 fake_encoder_.SetIsQpTrusted(true);
8506 // Enable quality scaling in encoder config.
8507 video_encoder_config.is_quality_scaling_allowed = true;
8508 ConfigureEncoder(std::move(video_encoder_config));
8509
8510 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008511 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008512
8513 test::FrameForwarder source;
8514 video_stream_encoder_->SetSource(
8515 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8516 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8517 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8518
8519 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8520 WaitForEncodedFrame(1);
8521 video_stream_encoder_->TriggerQualityLow();
8522 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8523
8524 video_stream_encoder_->Stop();
8525}
Shuhai Pengf2707702021-09-29 17:19:44 +08008526
8527TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8528 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8529
8530 // Disable scaling settings in encoder info.
8531 fake_encoder_.SetQualityScaling(false);
8532 // Set QP not trusted in encoder info.
8533 fake_encoder_.SetIsQpTrusted(false);
8534 // Enable quality scaling in encoder config.
8535 video_encoder_config.is_quality_scaling_allowed = true;
8536 ConfigureEncoder(std::move(video_encoder_config));
8537
8538 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008539 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008540
8541 test::FrameForwarder source;
8542 video_stream_encoder_->SetSource(
8543 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8544 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8545 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8546
8547 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8548 WaitForEncodedFrame(1);
8549 video_stream_encoder_->TriggerQualityLow();
8550 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8551 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8552 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8553
8554 video_stream_encoder_->Stop();
8555}
8556
8557TEST_F(VideoStreamEncoderTest,
8558 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8559 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8560
8561 // Disable scaling settings in encoder info.
8562 fake_encoder_.SetQualityScaling(false);
8563 // Set QP trusted in encoder info.
8564 fake_encoder_.SetIsQpTrusted(true);
8565 // Enable quality scaling in encoder config.
8566 video_encoder_config.is_quality_scaling_allowed = true;
8567 ConfigureEncoder(std::move(video_encoder_config));
8568
8569 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008570 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008571
8572 test::FrameForwarder source;
8573 video_stream_encoder_->SetSource(
8574 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8575 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8576 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8577
8578 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8579 WaitForEncodedFrame(1);
8580 video_stream_encoder_->TriggerQualityLow();
8581 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8582 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8583
8584 video_stream_encoder_->Stop();
8585}
8586
8587TEST_F(VideoStreamEncoderTest,
8588 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8589 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8590
8591 // Disable scaling settings in encoder info.
8592 fake_encoder_.SetQualityScaling(false);
8593 // Set QP trusted in encoder info.
8594 fake_encoder_.SetIsQpTrusted(false);
8595 // Enable quality scaling in encoder config.
8596 video_encoder_config.is_quality_scaling_allowed = true;
8597 ConfigureEncoder(std::move(video_encoder_config));
8598
8599 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008600 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008601
8602 test::FrameForwarder source;
8603 video_stream_encoder_->SetSource(
8604 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8605 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8606 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8607
8608 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8609 WaitForEncodedFrame(1);
8610 video_stream_encoder_->TriggerQualityLow();
8611 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8612
8613 video_stream_encoder_->Stop();
8614}
8615
Erik Språng5e13d052022-08-02 11:42:49 +02008616TEST_F(VideoStreamEncoderTest,
8617 RequestsRefreshFrameAfterEarlyDroppedNativeFrame) {
8618 // Send a native frame before encoder rates have been set. The encoder is
8619 // seen as paused at this time.
8620 rtc::Event frame_destroyed_event;
8621 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
8622 /*ntp_time_ms=*/1, &frame_destroyed_event, codec_width_, codec_height_));
8623
8624 // Frame should be dropped and destroyed.
8625 ExpectDroppedFrame();
Markus Handell2cfc1af2022-08-19 08:16:48 +00008626 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
Erik Språng5e13d052022-08-02 11:42:49 +02008627 EXPECT_EQ(video_source_.refresh_frames_requested_, 0);
8628
8629 // Set bitrates, unpausing the encoder and triggering a request for a refresh
8630 // frame.
8631 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8632 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8633 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8634 EXPECT_EQ(video_source_.refresh_frames_requested_, 1);
8635
8636 video_stream_encoder_->Stop();
8637}
8638
Erik Språnge4589cb2022-04-06 16:44:30 +02008639#endif // !defined(WEBRTC_IOS)
Sergey Silkind19e3b92021-03-16 10:05:30 +00008640
Henrik Boström56db9ff2021-03-24 09:06:45 +01008641// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8642class VideoStreamEncoderWithRealEncoderTest
8643 : public VideoStreamEncoderTest,
8644 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8645 public:
8646 VideoStreamEncoderWithRealEncoderTest()
8647 : VideoStreamEncoderTest(),
8648 codec_type_(std::get<0>(GetParam())),
8649 allow_i420_conversion_(std::get<1>(GetParam())) {}
8650
8651 void SetUp() override {
8652 VideoStreamEncoderTest::SetUp();
8653 std::unique_ptr<VideoEncoder> encoder;
8654 switch (codec_type_) {
8655 case kVideoCodecVP8:
8656 encoder = VP8Encoder::Create();
8657 break;
8658 case kVideoCodecVP9:
8659 encoder = VP9Encoder::Create();
8660 break;
8661 case kVideoCodecAV1:
philipel09a28482022-05-25 09:47:06 +02008662 encoder = CreateLibaomAv1Encoder();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008663 break;
8664 case kVideoCodecH264:
8665 encoder =
8666 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8667 break;
8668 case kVideoCodecMultiplex:
8669 mock_encoder_factory_for_multiplex_ =
8670 std::make_unique<MockVideoEncoderFactory>();
8671 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8672 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8673 .WillRepeatedly([] { return VP8Encoder::Create(); });
8674 encoder = std::make_unique<MultiplexEncoderAdapter>(
8675 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8676 false);
8677 break;
8678 default:
Artem Titovd3251962021-11-15 16:57:07 +01008679 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008680 }
8681 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8682 }
8683
8684 void TearDown() override {
8685 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 16:22:11 +02008686 // Ensure `video_stream_encoder_` is destroyed before
8687 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 09:06:45 +01008688 video_stream_encoder_.reset();
8689 VideoStreamEncoderTest::TearDown();
8690 }
8691
8692 protected:
8693 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8694 std::unique_ptr<VideoEncoder> encoder) {
8695 // Configure VSE to use the encoder.
8696 encoder_ = std::move(encoder);
8697 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8698 encoder_.get(), &encoder_selector_);
8699 video_send_config_.encoder_settings.encoder_factory =
8700 encoder_proxy_factory_.get();
8701 VideoEncoderConfig video_encoder_config;
8702 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8703 video_encoder_config_ = video_encoder_config.Copy();
8704 ConfigureEncoder(video_encoder_config_.Copy());
8705
8706 // Set bitrate to ensure frame is not dropped.
8707 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008708 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008709 }
8710
8711 const VideoCodecType codec_type_;
8712 const bool allow_i420_conversion_;
8713 NiceMock<MockEncoderSelector> encoder_selector_;
8714 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8715 std::unique_ptr<VideoEncoder> encoder_;
8716 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8717};
8718
8719TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8720 auto native_i420_frame = test::CreateMappableNativeFrame(
8721 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8722 video_source_.IncomingCapturedFrame(native_i420_frame);
8723 WaitForEncodedFrame(codec_width_, codec_height_);
8724
8725 auto mappable_native_buffer =
8726 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8727 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8728 mappable_native_buffer->GetMappedFramedBuffers();
8729 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8730 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8731 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8732 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8733}
8734
8735TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8736 auto native_nv12_frame = test::CreateMappableNativeFrame(
8737 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8738 video_source_.IncomingCapturedFrame(native_nv12_frame);
8739 WaitForEncodedFrame(codec_width_, codec_height_);
8740
8741 auto mappable_native_buffer =
8742 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8743 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8744 mappable_native_buffer->GetMappedFramedBuffers();
8745 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8746 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8747 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8748 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8749
8750 if (!allow_i420_conversion_) {
8751 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8752 }
8753}
8754
Erik Språng7444b192021-06-02 14:02:13 +02008755TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8756 if (codec_type_ == kVideoCodecMultiplex) {
8757 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8758 return;
8759 }
8760
8761 const size_t kNumSpatialLayers = 3u;
8762 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8763 const int kFrameWidth = 1280;
8764 const int kFrameHeight = 720;
8765 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8766 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8767 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8768 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8769 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8770 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8771
8772 VideoEncoderConfig config;
8773 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8774 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008775 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008776 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8777 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8778 vp9_settings.numberOfTemporalLayers = 3;
8779 vp9_settings.automaticResizeOn = false;
8780 config.encoder_specific_settings =
8781 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8782 vp9_settings);
8783 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8784 /*fps=*/30.0,
8785 /*first_active_layer=*/0,
8786 /*num_spatial_layers=*/3,
8787 /*num_temporal_layers=*/3,
8788 /*is_screenshare=*/false);
8789 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8790 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008791 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008792 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8793 /*fps=*/30.0,
8794 /*first_active_layer=*/0,
8795 /*num_spatial_layers=*/3,
8796 /*num_temporal_layers=*/3,
8797 /*is_screenshare=*/false);
Niels Möller79d566b2022-04-29 11:03:13 +02008798 config.simulcast_layers[0].scalability_mode = ScalabilityMode::kL3T3_KEY;
Erik Språng7444b192021-06-02 14:02:13 +02008799 } else {
8800 // Simulcast for VP8/H264.
8801 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8802 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8803 config.simulcast_layers[i].scale_resolution_down_by =
8804 kDownscaleFactors[i];
8805 config.simulcast_layers[i].active = true;
8806 }
8807 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8808 // Turn off frame dropping to prevent flakiness.
Niels Möller807328f2022-05-12 16:16:39 +02008809 config.frame_drop_enabled = false;
Erik Språng7444b192021-06-02 14:02:13 +02008810 }
8811 }
8812
8813 auto set_layer_active = [&](int layer_idx, bool active) {
8814 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8815 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8816 config.spatial_layers[layer_idx].active = active;
8817 } else {
8818 config.simulcast_layers[layer_idx].active = active;
8819 }
8820 };
8821
8822 config.video_stream_factory =
8823 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8824 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8825 /*screencast*/ false,
8826 /*screenshare enabled*/ false);
8827 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008828 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8829 0, 0, 0);
Erik Språng7444b192021-06-02 14:02:13 +02008830
8831 // Capture a frame with all layers active.
8832 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8833 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8834 int64_t timestamp_ms = kFrameIntervalMs;
8835 video_source_.IncomingCapturedFrame(
8836 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8837
8838 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8839 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8840
8841 // Capture a frame with one of the layers inactive.
8842 set_layer_active(2, false);
8843 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8844 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8845 timestamp_ms += kFrameIntervalMs;
8846 video_source_.IncomingCapturedFrame(
8847 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8848 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8849
8850 // New target bitrates signaled based on lower resolution.
8851 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8852 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8853 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8854 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8855
8856 // Re-enable the top layer.
8857 set_layer_active(2, true);
8858 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8859 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8860 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8861
8862 // Bitrate target adjusted back up to enable HD layer...
8863 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8864 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8865 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8866 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8867
8868 // ...then add a new frame.
8869 timestamp_ms += kFrameIntervalMs;
8870 video_source_.IncomingCapturedFrame(
8871 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8872 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8873 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8874
8875 video_stream_encoder_->Stop();
8876}
8877
Henrik Boström56db9ff2021-03-24 09:06:45 +01008878std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8879 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8880 VideoCodecType codec_type = std::get<0>(info.param);
8881 bool allow_i420_conversion = std::get<1>(info.param);
8882 std::string str;
8883 switch (codec_type) {
8884 case kVideoCodecGeneric:
8885 str = "Generic";
8886 break;
8887 case kVideoCodecVP8:
8888 str = "VP8";
8889 break;
8890 case kVideoCodecVP9:
8891 str = "VP9";
8892 break;
8893 case kVideoCodecAV1:
8894 str = "AV1";
8895 break;
8896 case kVideoCodecH264:
8897 str = "H264";
8898 break;
8899 case kVideoCodecMultiplex:
8900 str = "Multiplex";
8901 break;
8902 default:
Artem Titovd3251962021-11-15 16:57:07 +01008903 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008904 }
8905 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8906 return str;
8907}
8908
8909constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8910 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8911constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8912 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8913constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
Ilya Nikolaevskiy1bcdafc2022-03-09 16:01:07 +01008914 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/false);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008915constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8916 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8917#if defined(WEBRTC_USE_H264)
8918constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8919 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8920
8921// The windows compiler does not tolerate #if statements inside the
8922// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8923// and without H264).
8924INSTANTIATE_TEST_SUITE_P(
8925 All,
8926 VideoStreamEncoderWithRealEncoderTest,
8927 ::testing::Values(kVP8DisallowConversion,
8928 kVP9DisallowConversion,
8929 kAV1AllowConversion,
8930 kMultiplexDisallowConversion,
8931 kH264AllowConversion),
8932 TestParametersVideoCodecAndAllowI420ConversionToString);
8933#else
8934INSTANTIATE_TEST_SUITE_P(
8935 All,
8936 VideoStreamEncoderWithRealEncoderTest,
8937 ::testing::Values(kVP8DisallowConversion,
8938 kVP9DisallowConversion,
8939 kAV1AllowConversion,
8940 kMultiplexDisallowConversion),
8941 TestParametersVideoCodecAndAllowI420ConversionToString);
8942#endif
8943
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008944class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8945 protected:
8946 void RunTest(const std::vector<VideoStream>& configs,
8947 const int expected_num_init_encode) {
8948 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008949 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008950 InsertFrameAndWaitForEncoded();
8951 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8952 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008953 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8954 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008955
8956 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8957 ConfigureEncoder(configs[1]);
8958 InsertFrameAndWaitForEncoded();
8959 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8960 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008961 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008962 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 10:07:11 +02008963 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008964
8965 video_stream_encoder_->Stop();
8966 }
8967
8968 void ConfigureEncoder(const VideoStream& stream) {
8969 VideoEncoderConfig config;
8970 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8971 config.max_bitrate_bps = stream.max_bitrate_bps;
8972 config.simulcast_layers[0] = stream;
8973 config.video_stream_factory =
8974 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8975 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8976 /*conference_mode=*/false);
8977 video_stream_encoder_->ConfigureEncoder(std::move(config),
8978 kMaxPayloadLength);
8979 }
8980
8981 void OnBitrateUpdated(DataRate bitrate) {
8982 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8983 bitrate, bitrate, bitrate, 0, 0, 0);
8984 }
8985
8986 void InsertFrameAndWaitForEncoded() {
8987 timestamp_ms_ += kFrameIntervalMs;
8988 video_source_.IncomingCapturedFrame(
8989 CreateFrame(timestamp_ms_, kWidth, kHeight));
8990 sink_.WaitForEncodedFrame(timestamp_ms_);
8991 }
8992
8993 void ExpectEqual(const VideoCodec& actual,
8994 const VideoStream& expected) const {
8995 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8996 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8997 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
8998 static_cast<unsigned int>(expected.min_bitrate_bps));
8999 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
9000 static_cast<unsigned int>(expected.max_bitrate_bps));
9001 EXPECT_EQ(actual.simulcastStream[0].width,
9002 kWidth / expected.scale_resolution_down_by);
9003 EXPECT_EQ(actual.simulcastStream[0].height,
9004 kHeight / expected.scale_resolution_down_by);
9005 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
9006 expected.num_temporal_layers);
Niels Möller79d566b2022-04-29 11:03:13 +02009007 EXPECT_EQ(actual.GetScalabilityMode(), expected.scalability_mode);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02009008 }
9009
9010 VideoStream DefaultConfig() const {
9011 VideoStream stream;
9012 stream.max_framerate = 25;
9013 stream.min_bitrate_bps = 35000;
9014 stream.max_bitrate_bps = 900000;
9015 stream.scale_resolution_down_by = 1.0;
9016 stream.num_temporal_layers = 1;
9017 stream.bitrate_priority = 1.0;
Niels Möller79d566b2022-04-29 11:03:13 +02009018 stream.scalability_mode = absl::nullopt;
Åsa Persson4d4f62f2021-09-12 16:14:48 +02009019 return stream;
9020 }
9021
9022 const int kWidth = 640;
9023 const int kHeight = 360;
9024 int64_t timestamp_ms_ = 0;
9025};
9026
9027TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
9028 VideoStream config1 = DefaultConfig();
9029 VideoStream config2 = config1;
9030 config2.max_framerate++;
9031
9032 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9033}
9034
9035TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
9036 VideoStream config1 = DefaultConfig();
9037 VideoStream config2 = config1;
9038 config2.min_bitrate_bps += 10000;
9039
9040 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9041}
9042
9043TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
9044 VideoStream config1 = DefaultConfig();
9045 VideoStream config2 = config1;
9046 config2.max_bitrate_bps += 100000;
9047
9048 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9049}
9050
9051TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
9052 VideoStream config1 = DefaultConfig();
9053 VideoStream config2 = config1;
9054 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
9055
9056 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9057}
9058
9059TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
9060 VideoStream config1 = DefaultConfig();
9061 VideoStream config2 = config1;
9062 config2.scale_resolution_down_by *= 2;
9063
9064 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9065}
9066
9067TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
9068 VideoStream config1 = DefaultConfig();
9069 VideoStream config2 = config1;
9070 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
9071
9072 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9073}
9074
9075TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
9076 VideoStream config1 = DefaultConfig();
9077 VideoStream config2 = config1;
Niels Möller05954892022-05-13 13:34:37 +02009078 config2.scalability_mode = ScalabilityMode::kL2T1;
9079
9080 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9081}
9082
9083TEST_F(ReconfigureEncoderTest,
9084 UpdatesNumTemporalLayersFromScalabilityModeChanges) {
9085 VideoStream config1 = DefaultConfig();
9086 VideoStream config2 = config1;
Niels Möller79d566b2022-04-29 11:03:13 +02009087 config2.scalability_mode = ScalabilityMode::kL1T2;
Niels Möller05954892022-05-13 13:34:37 +02009088 config2.num_temporal_layers = 2;
Åsa Persson4d4f62f2021-09-12 16:14:48 +02009089
9090 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9091}
9092
Tommi62b01db2022-01-25 23:41:22 +01009093// Simple test that just creates and then immediately destroys an encoder.
9094// The purpose of the test is to make sure that nothing bad happens if the
9095// initialization step on the encoder queue, doesn't run.
9096TEST(VideoStreamEncoderSimpleTest, CreateDestroy) {
9097 class SuperLazyTaskQueue : public webrtc::TaskQueueBase {
9098 public:
9099 SuperLazyTaskQueue() = default;
9100 ~SuperLazyTaskQueue() override = default;
9101
9102 private:
9103 void Delete() override { delete this; }
Danil Chapovalov95eeaa72022-07-06 10:14:29 +02009104 void PostTask(absl::AnyInvocable<void() &&> task) override {
Tommi62b01db2022-01-25 23:41:22 +01009105 // meh.
9106 }
Danil Chapovalov95eeaa72022-07-06 10:14:29 +02009107 void PostDelayedTask(absl::AnyInvocable<void() &&> task,
9108 TimeDelta delay) override {
Tommi62b01db2022-01-25 23:41:22 +01009109 ASSERT_TRUE(false);
9110 }
Danil Chapovalov95eeaa72022-07-06 10:14:29 +02009111 void PostDelayedHighPrecisionTask(absl::AnyInvocable<void() &&> task,
9112 TimeDelta delay) override {
9113 ADD_FAILURE();
9114 }
Tommi62b01db2022-01-25 23:41:22 +01009115 };
9116
9117 // Lots of boiler plate.
Jonas Oreland8ca06132022-03-14 12:52:48 +01009118 test::ScopedKeyValueConfig field_trials;
Philipp Hanckea204ad22022-07-08 18:43:25 +02009119 GlobalSimulatedTimeController time_controller(Timestamp::Zero());
Tommi62b01db2022-01-25 23:41:22 +01009120 auto stats_proxy = std::make_unique<MockableSendStatisticsProxy>(
9121 time_controller.GetClock(), VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 12:52:48 +01009122 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo, field_trials);
Tommi62b01db2022-01-25 23:41:22 +01009123 SimpleVideoStreamEncoderFactory::MockFakeEncoder mock_fake_encoder(
9124 time_controller.GetClock());
9125 test::VideoEncoderProxyFactory encoder_factory(&mock_fake_encoder);
9126 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory =
9127 CreateBuiltinVideoBitrateAllocatorFactory();
9128 VideoStreamEncoderSettings encoder_settings{
9129 VideoEncoder::Capabilities(/*loss_notification=*/false)};
9130 encoder_settings.encoder_factory = &encoder_factory;
9131 encoder_settings.bitrate_allocator_factory = bitrate_allocator_factory.get();
9132
9133 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9134 EXPECT_CALL((*adapter.get()), Initialize).WillOnce(Return());
9135
9136 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
9137 encoder_queue(new SuperLazyTaskQueue());
9138
9139 // Construct a VideoStreamEncoder instance and let it go out of scope without
9140 // doing anything else (including calling Stop()). This should be fine since
9141 // the posted init task will simply be deleted.
9142 auto encoder = std::make_unique<VideoStreamEncoder>(
9143 time_controller.GetClock(), 1, stats_proxy.get(), encoder_settings,
Markus Handell8e4197b2022-05-30 15:45:28 +02009144 std::make_unique<CpuOveruseDetectorProxy>(stats_proxy.get(),
9145 field_trials),
Tommi62b01db2022-01-25 23:41:22 +01009146 std::move(adapter), std::move(encoder_queue),
9147 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009148 kVideoBitrateAllocation,
9149 field_trials);
Tommi4d6dd172022-05-23 09:34:41 +02009150
9151 // Stop the encoder explicitly. This additional step tests if we could
9152 // hang when calling stop and the TQ has been stopped and/or isn't accepting
9153 // any more tasks.
9154 encoder->Stop();
Tommi62b01db2022-01-25 23:41:22 +01009155}
9156
Markus Handellb4e96d42021-11-05 12:00:55 +01009157TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
9158 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9159 auto* adapter_ptr = adapter.get();
9160 SimpleVideoStreamEncoderFactory factory;
Markus Handell8d87c462021-12-16 11:37:16 +01009161 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9162 nullptr;
9163 EXPECT_CALL(*adapter_ptr, Initialize)
9164 .WillOnce(Invoke([&video_stream_encoder_callback](
9165 FrameCadenceAdapterInterface::Callback* callback) {
9166 video_stream_encoder_callback = callback;
9167 }));
9168 TaskQueueBase* encoder_queue = nullptr;
9169 auto video_stream_encoder =
9170 factory.Create(std::move(adapter), &encoder_queue);
Markus Handellb4e96d42021-11-05 12:00:55 +01009171
Markus Handelle59fee82021-12-23 09:29:23 +01009172 // First a call before we know the frame size and hence cannot compute the
9173 // number of simulcast layers.
9174 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
9175 &FrameCadenceAdapterInterface::
9176 ZeroHertzModeParams::num_simulcast_layers,
Markus Handell5a77e512022-09-01 12:51:50 +00009177 Eq(0u)))));
Markus Handellb4e96d42021-11-05 12:00:55 +01009178 VideoEncoderConfig config;
Markus Handell8d87c462021-12-16 11:37:16 +01009179 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
Markus Handellb4e96d42021-11-05 12:00:55 +01009180 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9181 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
Markus Handelle59fee82021-12-23 09:29:23 +01009182 factory.DepleteTaskQueues();
9183
9184 // Then a call as we've computed the number of simulcast layers after a passed
9185 // frame.
9186 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
9187 &FrameCadenceAdapterInterface::
9188 ZeroHertzModeParams::num_simulcast_layers,
Markus Handell5a77e512022-09-01 12:51:50 +00009189 Gt(0u)))));
Markus Handell8d87c462021-12-16 11:37:16 +01009190 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handell9a478b52021-11-18 16:07:01 +01009191 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01009192 Mock::VerifyAndClearExpectations(adapter_ptr);
9193
Markus Handelle59fee82021-12-23 09:29:23 +01009194 // Expect a disabled zero-hertz mode after passing realtime video.
Markus Handell8d87c462021-12-16 11:37:16 +01009195 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Eq(absl::nullopt)));
Markus Handellb4e96d42021-11-05 12:00:55 +01009196 VideoEncoderConfig config2;
Markus Handell8d87c462021-12-16 11:37:16 +01009197 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config2);
Markus Handellb4e96d42021-11-05 12:00:55 +01009198 config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
9199 video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
Markus Handell8d87c462021-12-16 11:37:16 +01009200 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
Markus Handell9a478b52021-11-18 16:07:01 +01009201 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01009202}
9203
9204TEST(VideoStreamEncoderFrameCadenceTest,
9205 ForwardsFramesIntoFrameCadenceAdapter) {
9206 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9207 auto* adapter_ptr = adapter.get();
9208 test::FrameForwarder video_source;
9209 SimpleVideoStreamEncoderFactory factory;
9210 auto video_stream_encoder = factory.Create(std::move(adapter));
9211 video_stream_encoder->SetSource(
9212 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9213
9214 EXPECT_CALL(*adapter_ptr, OnFrame);
9215 auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
9216 video_source.IncomingCapturedFrame(
Markus Handell8d87c462021-12-16 11:37:16 +01009217 VideoFrame::Builder().set_video_frame_buffer(std::move(buffer)).build());
Markus Handellb4e96d42021-11-05 12:00:55 +01009218}
9219
Markus Handellee225432021-11-29 12:35:12 +01009220TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) {
9221 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9222 auto* adapter_ptr = adapter.get();
9223 test::FrameForwarder video_source;
9224 SimpleVideoStreamEncoderFactory factory;
9225 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9226 nullptr;
9227 EXPECT_CALL(*adapter_ptr, Initialize)
9228 .WillOnce(Invoke([&video_stream_encoder_callback](
9229 FrameCadenceAdapterInterface::Callback* callback) {
9230 video_stream_encoder_callback = callback;
9231 }));
9232 TaskQueueBase* encoder_queue = nullptr;
9233 auto video_stream_encoder =
9234 factory.Create(std::move(adapter), &encoder_queue);
9235
9236 // This is just to make the VSE operational. We'll feed a frame directly by
9237 // the callback interface.
9238 video_stream_encoder->SetSource(
9239 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9240
9241 VideoEncoderConfig video_encoder_config;
9242 test::FillEncoderConfiguration(kVideoCodecGeneric, 1, &video_encoder_config);
9243 video_stream_encoder->ConfigureEncoder(std::move(video_encoder_config),
9244 /*max_data_payload_length=*/1000);
9245
9246 EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps);
9247 EXPECT_CALL(*adapter_ptr, UpdateFrameRate);
Markus Handell8d87c462021-12-16 11:37:16 +01009248 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handellee225432021-11-29 12:35:12 +01009249 factory.DepleteTaskQueues();
9250}
9251
Markus Handell8d87c462021-12-16 11:37:16 +01009252TEST(VideoStreamEncoderFrameCadenceTest,
9253 DeactivatesActivatesLayersOnBitrateChanges) {
9254 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9255 auto* adapter_ptr = adapter.get();
9256 SimpleVideoStreamEncoderFactory factory;
9257 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9258 nullptr;
9259 EXPECT_CALL(*adapter_ptr, Initialize)
9260 .WillOnce(Invoke([&video_stream_encoder_callback](
9261 FrameCadenceAdapterInterface::Callback* callback) {
9262 video_stream_encoder_callback = callback;
9263 }));
9264 TaskQueueBase* encoder_queue = nullptr;
9265 auto video_stream_encoder =
9266 factory.Create(std::move(adapter), &encoder_queue);
9267
9268 // Configure 2 simulcast layers. FillEncoderConfiguration sets min bitrates to
9269 // {150000, 450000}.
9270 VideoEncoderConfig video_encoder_config;
9271 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9272 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9273 kMaxPayloadLength);
9274 // Ensure an encoder is created.
9275 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9276
9277 // Both layers enabled at 1 MBit/s.
9278 video_stream_encoder->OnBitrateUpdated(
9279 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9280 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9281 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9282 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9283 factory.DepleteTaskQueues();
9284 Mock::VerifyAndClearExpectations(adapter_ptr);
9285
9286 // Layer 1 disabled at 200 KBit/s.
9287 video_stream_encoder->OnBitrateUpdated(
9288 DataRate::KilobitsPerSec(200), DataRate::KilobitsPerSec(200),
9289 DataRate::KilobitsPerSec(200), 0, 0, 0);
9290 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9291 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9292 factory.DepleteTaskQueues();
9293 Mock::VerifyAndClearExpectations(adapter_ptr);
9294
9295 // All layers off at suspended video.
9296 video_stream_encoder->OnBitrateUpdated(DataRate::Zero(), DataRate::Zero(),
9297 DataRate::Zero(), 0, 0, 0);
9298 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/false));
9299 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9300 factory.DepleteTaskQueues();
9301 Mock::VerifyAndClearExpectations(adapter_ptr);
9302
9303 // Both layers enabled again back at 1 MBit/s.
9304 video_stream_encoder->OnBitrateUpdated(
9305 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9306 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9307 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9308 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9309 factory.DepleteTaskQueues();
9310}
9311
9312TEST(VideoStreamEncoderFrameCadenceTest, UpdatesQualityConvergence) {
9313 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9314 auto* adapter_ptr = adapter.get();
9315 SimpleVideoStreamEncoderFactory factory;
9316 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9317 nullptr;
9318 EXPECT_CALL(*adapter_ptr, Initialize)
9319 .WillOnce(Invoke([&video_stream_encoder_callback](
9320 FrameCadenceAdapterInterface::Callback* callback) {
9321 video_stream_encoder_callback = callback;
9322 }));
9323 TaskQueueBase* encoder_queue = nullptr;
9324 auto video_stream_encoder =
9325 factory.Create(std::move(adapter), &encoder_queue);
9326
9327 // Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder.
9328 VideoEncoderConfig video_encoder_config;
9329 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9330 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9331 kMaxPayloadLength);
9332 video_stream_encoder->OnBitrateUpdated(
9333 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9334 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9335
9336 // Pass a frame which has unconverged results.
9337 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9338 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9339 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9340 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
9341 EXPECT_FALSE(encoded_image.IsAtTargetQuality());
9342 CodecSpecificInfo codec_specific;
9343 codec_specific.codecType = kVideoCodecGeneric;
9344 return codec_specific;
9345 }));
9346 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, false));
9347 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9348 factory.DepleteTaskQueues();
9349 Mock::VerifyAndClearExpectations(adapter_ptr);
9350 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9351
9352 // Pass a frame which converges in layer 0 and not in layer 1.
9353 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9354 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9355 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9356 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
9357 encoded_image.SetAtTargetQuality(encoded_image.SpatialIndex() == 0);
9358 CodecSpecificInfo codec_specific;
9359 codec_specific.codecType = kVideoCodecGeneric;
9360 return codec_specific;
9361 }));
9362 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, true));
9363 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9364 factory.DepleteTaskQueues();
9365 Mock::VerifyAndClearExpectations(adapter_ptr);
9366 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9367}
9368
Markus Handell2e0f4f02021-12-21 19:14:58 +01009369TEST(VideoStreamEncoderFrameCadenceTest,
9370 RequestsRefreshFramesWhenCadenceAdapterInstructs) {
9371 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9372 auto* adapter_ptr = adapter.get();
9373 MockVideoSourceInterface mock_source;
9374 SimpleVideoStreamEncoderFactory factory;
9375 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9376 nullptr;
9377 EXPECT_CALL(*adapter_ptr, Initialize)
9378 .WillOnce(Invoke([&video_stream_encoder_callback](
9379 FrameCadenceAdapterInterface::Callback* callback) {
9380 video_stream_encoder_callback = callback;
9381 }));
9382 TaskQueueBase* encoder_queue = nullptr;
9383 auto video_stream_encoder =
9384 factory.Create(std::move(adapter), &encoder_queue);
9385 video_stream_encoder->SetSource(
9386 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9387 VideoEncoderConfig config;
9388 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9389 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9390 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9391 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9392 // Ensure the encoder is set up.
9393 factory.DepleteTaskQueues();
9394
Markus Handell818e7fb2021-12-30 13:01:33 +01009395 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest)
9396 .WillOnce(Invoke([video_stream_encoder_callback] {
9397 video_stream_encoder_callback->RequestRefreshFrame();
9398 }));
Markus Handell2e0f4f02021-12-21 19:14:58 +01009399 EXPECT_CALL(mock_source, RequestRefreshFrame);
9400 video_stream_encoder->SendKeyFrame();
9401 factory.DepleteTaskQueues();
9402 Mock::VerifyAndClearExpectations(adapter_ptr);
9403 Mock::VerifyAndClearExpectations(&mock_source);
9404
Markus Handell818e7fb2021-12-30 13:01:33 +01009405 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest);
Markus Handell2e0f4f02021-12-21 19:14:58 +01009406 EXPECT_CALL(mock_source, RequestRefreshFrame).Times(0);
9407 video_stream_encoder->SendKeyFrame();
9408 factory.DepleteTaskQueues();
9409}
9410
Markus Handell818e7fb2021-12-30 13:01:33 +01009411TEST(VideoStreamEncoderFrameCadenceTest,
9412 RequestsRefreshFrameForEarlyZeroHertzKeyFrameRequest) {
9413 SimpleVideoStreamEncoderFactory factory;
9414 auto encoder_queue =
9415 factory.GetTimeController()->GetTaskQueueFactory()->CreateTaskQueue(
9416 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
9417
9418 // Enables zero-hertz mode.
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009419 test::ScopedKeyValueConfig field_trials(
9420 "WebRTC-ZeroHertzScreenshare/Enabled/");
Markus Handell818e7fb2021-12-30 13:01:33 +01009421 auto adapter = FrameCadenceAdapterInterface::Create(
Jonas Oreland8ca06132022-03-14 12:52:48 +01009422 factory.GetTimeController()->GetClock(), encoder_queue.get(),
9423 field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009424 FrameCadenceAdapterInterface* adapter_ptr = adapter.get();
9425
9426 MockVideoSourceInterface mock_source;
9427 auto video_stream_encoder = factory.CreateWithEncoderQueue(
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009428 std::move(adapter), std::move(encoder_queue), &field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009429
9430 video_stream_encoder->SetSource(
9431 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9432 VideoEncoderConfig config;
9433 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9434 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9435 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9436
9437 // Eventually expect a refresh frame request when requesting a key frame
9438 // before initializing zero-hertz mode. This can happen in reality because the
9439 // threads invoking key frame requests and constraints setup aren't
9440 // synchronized.
9441 EXPECT_CALL(mock_source, RequestRefreshFrame);
9442 video_stream_encoder->SendKeyFrame();
Markus Handellf5a50792022-06-16 14:25:15 +02009443 constexpr int kMaxFps = 30;
9444 adapter_ptr->OnConstraintsChanged(VideoTrackSourceConstraints{0, kMaxFps});
9445 factory.GetTimeController()->AdvanceTime(
9446 TimeDelta::Seconds(1) *
9447 FrameCadenceAdapterInterface::kOnDiscardedFrameRefreshFramePeriod /
9448 kMaxFps);
Markus Handell818e7fb2021-12-30 13:01:33 +01009449}
9450
perkj26091b12016-09-01 01:17:40 -07009451} // namespace webrtc