blob: 575a8e24f527b170b9b663f6272e20266c0f0d71 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Markus Handell8e4197b2022-05-30 15:45:28 +020020#include "api/field_trials_view.h"
Markus Handell9a478b52021-11-18 16:07:01 +010021#include "api/rtp_parameters.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020022#include "api/task_queue/default_task_queue_factory.h"
Markus Handell818e7fb2021-12-30 13:01:33 +010023#include "api/task_queue/task_queue_base.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010024#include "api/task_queue/task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020025#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010026#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010027#include "api/test/mock_video_encoder_factory.h"
Markus Handell8d87c462021-12-16 11:37:16 +010028#include "api/units/data_rate.h"
Markus Handellf5a50792022-06-16 14:25:15 +020029#include "api/units/time_delta.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080030#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020031#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020032#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020033#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010034#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010035#include "api/video_codecs/sdp_video_format.h"
Erik Språnge4589cb2022-04-06 16:44:30 +020036#include "api/video_codecs/video_codec.h"
Elad Alon370f93a2019-06-11 14:57:57 +020037#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020038#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010039#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020040#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010041#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020042#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070043#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080044#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020045#include "media/engine/webrtc_video_engine.h"
philipel09a28482022-05-25 09:47:06 +020046#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010047#include "modules/video_coding/codecs/h264/include/h264.h"
48#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
49#include "modules/video_coding/codecs/vp8/include/vp8.h"
50#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020051#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020052#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020053#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010054#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020055#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010056#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020057#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020058#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080059#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020060#include "rtc_base/synchronization/mutex.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020061#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020062#include "test/encoder_settings.h"
63#include "test/fake_encoder.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010064#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020065#include "test/gmock.h"
66#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010067#include "test/mappable_native_buffer.h"
Jonas Orelandc7f691a2022-03-09 15:12:07 +010068#include "test/scoped_key_value_config.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020069#include "test/time_controller/simulated_time_controller.h"
Byoungchan Lee13fe3672022-04-06 10:44:42 +090070#include "test/video_encoder_nullable_proxy_factory.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020071#include "test/video_encoder_proxy_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010072#include "video/frame_cadence_adapter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020073#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070074
75namespace webrtc {
76
sprang57c2fff2017-01-16 06:24:02 -080077using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020078using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020079using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020080using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020081using ::testing::Ge;
82using ::testing::Gt;
Markus Handellee225432021-11-29 12:35:12 +010083using ::testing::Invoke;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020084using ::testing::Le;
85using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010086using ::testing::Matcher;
Markus Handellb4e96d42021-11-05 12:00:55 +010087using ::testing::Mock;
philipel9b058032020-02-10 11:30:00 +010088using ::testing::NiceMock;
Markus Handellb4e96d42021-11-05 12:00:55 +010089using ::testing::Optional;
philipel9b058032020-02-10 11:30:00 +010090using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010091using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020092using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080093
perkj803d97f2016-11-01 11:45:46 -070094namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020095const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010096const int kQpLow = 1;
97const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020098const int kMinFramerateFps = 2;
99const int kMinBalancedFramerateFps = 7;
100const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -0800101const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 10:07:11 +0200102const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
103const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
104const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
105const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 07:02:22 -0800106const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -0700107const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200108const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +0200109const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200110const VideoEncoder::ResolutionBitrateLimits
111 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
112const VideoEncoder::ResolutionBitrateLimits
113 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800114
Asa Persson606d3cb2021-10-04 10:07:11 +0200115uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200116 0x00, 0x00, 0x03, 0x03, 0xF4,
117 0x05, 0x03, 0xC7, 0xE0, 0x1B,
118 0x41, 0x10, 0x8D, 0x00};
119
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100120const uint8_t kCodedFrameVp8Qp25[] = {
121 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
122 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
123 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
124
Markus Handell818e7fb2021-12-30 13:01:33 +0100125VideoFrame CreateSimpleNV12Frame() {
126 return VideoFrame::Builder()
127 .set_video_frame_buffer(rtc::make_ref_counted<NV12Buffer>(
128 /*width=*/16, /*height=*/16))
129 .build();
130}
131
Markus Handell8d87c462021-12-16 11:37:16 +0100132void PassAFrame(
133 TaskQueueBase* encoder_queue,
134 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback,
135 int64_t ntp_time_ms) {
136 encoder_queue->PostTask(
137 ToQueuedTask([video_stream_encoder_callback, ntp_time_ms] {
Markus Handell818e7fb2021-12-30 13:01:33 +0100138 video_stream_encoder_callback->OnFrame(Timestamp::Millis(ntp_time_ms),
139 1, CreateSimpleNV12Frame());
Markus Handell8d87c462021-12-16 11:37:16 +0100140 }));
141}
142
perkj803d97f2016-11-01 11:45:46 -0700143class TestBuffer : public webrtc::I420Buffer {
144 public:
145 TestBuffer(rtc::Event* event, int width, int height)
146 : I420Buffer(width, height), event_(event) {}
147
148 private:
149 friend class rtc::RefCountedObject<TestBuffer>;
150 ~TestBuffer() override {
151 if (event_)
152 event_->Set();
153 }
154 rtc::Event* const event_;
155};
156
Henrik Boström56db9ff2021-03-24 09:06:45 +0100157// A fake native buffer that can't be converted to I420. Upon scaling, it
158// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700159class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
160 public:
161 FakeNativeBuffer(rtc::Event* event, int width, int height)
162 : event_(event), width_(width), height_(height) {}
163 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
164 int width() const override { return width_; }
165 int height() const override { return height_; }
166 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
167 return nullptr;
168 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100169 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
170 int offset_x,
171 int offset_y,
172 int crop_width,
173 int crop_height,
174 int scaled_width,
175 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200176 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
177 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100178 }
Noah Richards51db4212019-06-12 06:59:12 -0700179
180 private:
181 friend class rtc::RefCountedObject<FakeNativeBuffer>;
182 ~FakeNativeBuffer() override {
183 if (event_)
184 event_->Set();
185 }
186 rtc::Event* const event_;
187 const int width_;
188 const int height_;
189};
190
Evan Shrubsole895556e2020-10-05 09:15:13 +0200191// A fake native buffer that is backed by an NV12 buffer.
192class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
193 public:
194 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
195 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
196
197 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
198 int width() const override { return nv12_buffer_->width(); }
199 int height() const override { return nv12_buffer_->height(); }
200 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
201 return nv12_buffer_->ToI420();
202 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200203 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
204 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
205 if (absl::c_find(types, Type::kNV12) != types.end()) {
206 return nv12_buffer_;
207 }
208 return nullptr;
209 }
Niels Möllerba2de582022-04-20 16:46:26 +0200210 const NV12BufferInterface* GetNV12() const { return nv12_buffer_.get(); }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200211
212 private:
213 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
214 ~FakeNV12NativeBuffer() override {
215 if (event_)
216 event_->Set();
217 }
218 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
219 rtc::Event* const event_;
220};
221
Niels Möller7dc26b72017-12-06 10:27:48 +0100222class CpuOveruseDetectorProxy : public OveruseFrameDetector {
223 public:
Markus Handell8e4197b2022-05-30 15:45:28 +0200224 CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer,
225 const FieldTrialsView& field_trials)
226 : OveruseFrameDetector(metrics_observer, field_trials),
Henrik Boström381d1092020-05-12 18:49:07 +0200227 last_target_framerate_fps_(-1),
228 framerate_updated_event_(true /* manual_reset */,
229 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100230 virtual ~CpuOveruseDetectorProxy() {}
231
232 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200233 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100234 last_target_framerate_fps_ = framerate_fps;
235 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200236 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100237 }
238
239 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200240 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100241 return last_target_framerate_fps_;
242 }
243
Niels Möller4db138e2018-04-19 09:04:13 +0200244 CpuOveruseOptions GetOptions() { return options_; }
245
Henrik Boström381d1092020-05-12 18:49:07 +0200246 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
247
Niels Möller7dc26b72017-12-06 10:27:48 +0100248 private:
Markus Handella3765182020-07-08 13:13:32 +0200249 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100250 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200251 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100252};
253
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200254class FakeVideoSourceRestrictionsListener
255 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200256 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200257 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200258 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200259 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200260 RTC_DCHECK(was_restrictions_updated_);
261 }
262
263 rtc::Event* restrictions_updated_event() {
264 return &restrictions_updated_event_;
265 }
266
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200267 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200268 void OnVideoSourceRestrictionsUpdated(
269 VideoSourceRestrictions restrictions,
270 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200271 rtc::scoped_refptr<Resource> reason,
272 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200273 was_restrictions_updated_ = true;
274 restrictions_updated_event_.Set();
275 }
276
277 private:
278 bool was_restrictions_updated_;
279 rtc::Event restrictions_updated_event_;
280};
281
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200282auto WantsFps(Matcher<int> fps_matcher) {
283 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
284 fps_matcher);
285}
286
287auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
288 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
289 AllOf(max_pixel_matcher, Gt(0)));
290}
291
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200292auto ResolutionMax() {
293 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200294 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200295 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
296 Eq(absl::nullopt)));
297}
298
299auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200300 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200301}
302
303auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200304 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200305}
306
307auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200308 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200309}
310
311auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200312 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200313}
314
315auto FpsMaxResolutionMax() {
316 return AllOf(FpsMax(), ResolutionMax());
317}
318
319auto UnlimitedSinkWants() {
320 return AllOf(FpsUnlimited(), ResolutionMax());
321}
322
323auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
324 Matcher<int> fps_range_matcher;
325
326 if (last_frame_pixels <= 320 * 240) {
327 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200328 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200329 fps_range_matcher = AllOf(Ge(10), Le(15));
330 } else if (last_frame_pixels <= 640 * 480) {
331 fps_range_matcher = Ge(15);
332 } else {
333 fps_range_matcher = Eq(kDefaultFramerate);
334 }
335 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
336 fps_range_matcher);
337}
338
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200339auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
340 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
341 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
342}
343
344auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
345 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
346}
347
348auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
349 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
350}
351
352auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
353 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
354 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
355}
356
357auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
358 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
359 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
360}
361
362auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
363 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
364 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
365}
366
367auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
368 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
369 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
370}
371
mflodmancc3d4422017-08-03 08:27:51 -0700372class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700373 public:
Markus Handell9a478b52021-11-18 16:07:01 +0100374 VideoStreamEncoderUnderTest(
375 TimeController* time_controller,
376 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter,
377 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
378 encoder_queue,
379 SendStatisticsProxy* stats_proxy,
380 const VideoStreamEncoderSettings& settings,
381 VideoStreamEncoder::BitrateAllocationCallbackType
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100382 allocation_callback_type,
Erik Språnge4589cb2022-04-06 16:44:30 +0200383 const FieldTrialsView& field_trials,
384 int num_cores)
Markus Handell8e4197b2022-05-30 15:45:28 +0200385 : VideoStreamEncoder(
386 time_controller->GetClock(),
387 num_cores,
388 stats_proxy,
389 settings,
390 std::unique_ptr<OveruseFrameDetector>(
391 overuse_detector_proxy_ =
392 new CpuOveruseDetectorProxy(stats_proxy, field_trials)),
393 std::move(cadence_adapter),
394 std::move(encoder_queue),
395 allocation_callback_type,
396 field_trials),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200397 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200398 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200399 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200400 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200401 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200402 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200403 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200404 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100405 }
perkj803d97f2016-11-01 11:45:46 -0700406
Henrik Boström381d1092020-05-12 18:49:07 +0200407 void SetSourceAndWaitForRestrictionsUpdated(
408 rtc::VideoSourceInterface<VideoFrame>* source,
409 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200410 FakeVideoSourceRestrictionsListener listener;
411 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200412 SetSource(source, degradation_preference);
413 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200414 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200415 }
416
417 void SetSourceAndWaitForFramerateUpdated(
418 rtc::VideoSourceInterface<VideoFrame>* source,
419 const DegradationPreference& degradation_preference) {
420 overuse_detector_proxy_->framerate_updated_event()->Reset();
421 SetSource(source, degradation_preference);
422 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
423 }
424
425 void OnBitrateUpdatedAndWaitForManagedResources(
426 DataRate target_bitrate,
427 DataRate stable_target_bitrate,
428 DataRate link_allocation,
429 uint8_t fraction_lost,
430 int64_t round_trip_time_ms,
431 double cwnd_reduce_ratio) {
432 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
433 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
434 // Bitrate is updated on the encoder queue.
435 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200436 }
437
kthelgason2fc52542017-03-03 00:24:41 -0800438 // This is used as a synchronisation mechanism, to make sure that the
439 // encoder queue is not blocked before we start sending it frames.
440 void WaitUntilTaskQueueIsIdle() {
Markus Handell28c71802021-11-08 10:11:55 +0100441 time_controller_->AdvanceTime(TimeDelta::Zero());
kthelgason2fc52542017-03-03 00:24:41 -0800442 }
443
Henrik Boström91aa7322020-04-28 12:24:33 +0200444 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200445 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200446 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200447 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200448 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200449 event.Set();
450 });
451 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100452 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200453 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200454
Henrik Boström91aa7322020-04-28 12:24:33 +0200455 void TriggerCpuUnderuse() {
456 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200457 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200458 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200459 event.Set();
460 });
461 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100462 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200463 }
kthelgason876222f2016-11-29 01:44:11 -0800464
Henrik Boström91aa7322020-04-28 12:24:33 +0200465 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200466 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200467 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200468 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200469 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200470 event.Set();
471 });
472 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100473 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200474 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200475 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200476 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200477 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200478 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200479 event.Set();
480 });
481 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100482 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200483 }
484
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200485 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100486 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200487 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
488 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200489 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700490};
491
Noah Richards51db4212019-06-12 06:59:12 -0700492// Simulates simulcast behavior and makes highest stream resolutions divisible
493// by 4.
494class CroppingVideoStreamFactory
495 : public VideoEncoderConfig::VideoStreamFactoryInterface {
496 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200497 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700498
499 private:
500 std::vector<VideoStream> CreateEncoderStreams(
501 int width,
502 int height,
503 const VideoEncoderConfig& encoder_config) override {
504 std::vector<VideoStream> streams = test::CreateVideoStreams(
505 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700506 return streams;
507 }
Noah Richards51db4212019-06-12 06:59:12 -0700508};
509
sprangb1ca0732017-02-01 08:38:12 -0800510class AdaptingFrameForwarder : public test::FrameForwarder {
511 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200512 explicit AdaptingFrameForwarder(TimeController* time_controller)
513 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700514 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800515
516 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200517 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800518 adaptation_enabled_ = enabled;
519 }
520
asaperssonfab67072017-04-04 05:51:49 -0700521 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200522 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800523 return adaptation_enabled_;
524 }
525
Henrik Boström1124ed12021-02-25 10:30:39 +0100526 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
527 // the resolution or frame rate was different than it is currently. If
528 // something else is modified, such as encoder resolutions, but the resolution
529 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700530 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200531 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700532 return last_wants_;
533 }
534
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200535 absl::optional<int> last_sent_width() const { return last_width_; }
536 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800537
sprangb1ca0732017-02-01 08:38:12 -0800538 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200539 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +0100540 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200541
sprangb1ca0732017-02-01 08:38:12 -0800542 int cropped_width = 0;
543 int cropped_height = 0;
544 int out_width = 0;
545 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700546 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000547 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
548 << "w=" << video_frame.width()
549 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700550 if (adapter_.AdaptFrameResolution(
551 video_frame.width(), video_frame.height(),
552 video_frame.timestamp_us() * 1000, &cropped_width,
553 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100554 VideoFrame adapted_frame =
555 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200556 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100557 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200558 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100559 .set_timestamp_ms(99)
560 .set_rotation(kVideoRotation_0)
561 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100562 if (video_frame.has_update_rect()) {
563 adapted_frame.set_update_rect(
564 video_frame.update_rect().ScaleWithFrame(
565 video_frame.width(), video_frame.height(), 0, 0,
566 video_frame.width(), video_frame.height(), out_width,
567 out_height));
568 }
sprangc5d62e22017-04-02 23:53:04 -0700569 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800570 last_width_.emplace(adapted_frame.width());
571 last_height_.emplace(adapted_frame.height());
572 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200573 last_width_ = absl::nullopt;
574 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700575 }
sprangb1ca0732017-02-01 08:38:12 -0800576 } else {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000577 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800578 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800579 last_width_.emplace(video_frame.width());
580 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800581 }
582 }
583
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200584 void OnOutputFormatRequest(int width, int height) {
585 absl::optional<std::pair<int, int>> target_aspect_ratio =
586 std::make_pair(width, height);
587 absl::optional<int> max_pixel_count = width * height;
588 absl::optional<int> max_fps;
589 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
590 max_fps);
591 }
592
sprangb1ca0732017-02-01 08:38:12 -0800593 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
594 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200595 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100596 rtc::VideoSinkWants prev_wants = sink_wants_locked();
597 bool did_adapt =
598 prev_wants.max_pixel_count != wants.max_pixel_count ||
599 prev_wants.target_pixel_count != wants.target_pixel_count ||
600 prev_wants.max_framerate_fps != wants.max_framerate_fps;
601 if (did_adapt) {
602 last_wants_ = prev_wants;
603 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100604 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200605 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800606 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200607
608 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800609 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200610 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
611 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200612 absl::optional<int> last_width_;
613 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800614};
sprangc5d62e22017-04-02 23:53:04 -0700615
Niels Möller213618e2018-07-24 09:29:58 +0200616// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700617class MockableSendStatisticsProxy : public SendStatisticsProxy {
618 public:
619 MockableSendStatisticsProxy(Clock* clock,
620 const VideoSendStream::Config& config,
Jonas Oreland8ca06132022-03-14 12:52:48 +0100621 VideoEncoderConfig::ContentType content_type,
Jonas Orelande62c2f22022-03-29 11:04:48 +0200622 const FieldTrialsView& field_trials)
Jonas Oreland8ca06132022-03-14 12:52:48 +0100623 : SendStatisticsProxy(clock, config, content_type, field_trials) {}
sprangc5d62e22017-04-02 23:53:04 -0700624
625 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200626 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700627 if (mock_stats_)
628 return *mock_stats_;
629 return SendStatisticsProxy::GetStats();
630 }
631
Niels Möller213618e2018-07-24 09:29:58 +0200632 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200633 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200634 if (mock_stats_)
635 return mock_stats_->input_frame_rate;
636 return SendStatisticsProxy::GetInputFrameRate();
637 }
sprangc5d62e22017-04-02 23:53:04 -0700638 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200639 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700640 mock_stats_.emplace(stats);
641 }
642
643 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200644 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700645 mock_stats_.reset();
646 }
647
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200648 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
649 on_frame_dropped_ = std::move(callback);
650 }
651
sprangc5d62e22017-04-02 23:53:04 -0700652 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200653 void OnFrameDropped(DropReason reason) override {
654 SendStatisticsProxy::OnFrameDropped(reason);
655 if (on_frame_dropped_)
656 on_frame_dropped_(reason);
657 }
658
Markus Handella3765182020-07-08 13:13:32 +0200659 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200660 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200661 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700662};
663
Markus Handellb4e96d42021-11-05 12:00:55 +0100664class SimpleVideoStreamEncoderFactory {
665 public:
666 class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
667 public:
668 using VideoStreamEncoder::VideoStreamEncoder;
669 ~AdaptedVideoStreamEncoder() { Stop(); }
670 };
671
Markus Handell8d87c462021-12-16 11:37:16 +0100672 class MockFakeEncoder : public test::FakeEncoder {
673 public:
674 using FakeEncoder::FakeEncoder;
675 MOCK_METHOD(CodecSpecificInfo,
676 EncodeHook,
677 (EncodedImage & encoded_image,
678 rtc::scoped_refptr<EncodedImageBuffer> buffer),
679 (override));
680 };
681
Markus Handellee225432021-11-29 12:35:12 +0100682 SimpleVideoStreamEncoderFactory() {
Markus Handellb4e96d42021-11-05 12:00:55 +0100683 encoder_settings_.encoder_factory = &encoder_factory_;
Markus Handellee225432021-11-29 12:35:12 +0100684 encoder_settings_.bitrate_allocator_factory =
685 bitrate_allocator_factory_.get();
Markus Handellb4e96d42021-11-05 12:00:55 +0100686 }
687
Markus Handell818e7fb2021-12-30 13:01:33 +0100688 std::unique_ptr<AdaptedVideoStreamEncoder> CreateWithEncoderQueue(
Markus Handellee225432021-11-29 12:35:12 +0100689 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100690 std::unique_ptr<TaskQueueBase, TaskQueueDeleter> encoder_queue,
Jonas Orelande62c2f22022-03-29 11:04:48 +0200691 const FieldTrialsView* field_trials = nullptr) {
Markus Handellb4e96d42021-11-05 12:00:55 +0100692 auto result = std::make_unique<AdaptedVideoStreamEncoder>(
693 time_controller_.GetClock(),
694 /*number_of_cores=*/1,
695 /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
Markus Handell8e4197b2022-05-30 15:45:28 +0200696 std::make_unique<CpuOveruseDetectorProxy>(
697 /*stats_proxy=*/nullptr,
698 field_trials ? *field_trials : field_trials_),
Markus Handellee225432021-11-29 12:35:12 +0100699 std::move(zero_hertz_adapter), std::move(encoder_queue),
Markus Handellb4e96d42021-11-05 12:00:55 +0100700 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100701 kVideoBitrateAllocation,
702 field_trials ? *field_trials : field_trials_);
Markus Handellb4e96d42021-11-05 12:00:55 +0100703 result->SetSink(&sink_, /*rotation_applied=*/false);
704 return result;
705 }
706
Markus Handell818e7fb2021-12-30 13:01:33 +0100707 std::unique_ptr<AdaptedVideoStreamEncoder> Create(
708 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
709 TaskQueueBase** encoder_queue_ptr = nullptr) {
710 auto encoder_queue =
711 time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
712 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
713 if (encoder_queue_ptr)
714 *encoder_queue_ptr = encoder_queue.get();
715 return CreateWithEncoderQueue(std::move(zero_hertz_adapter),
716 std::move(encoder_queue));
717 }
718
Markus Handell9a478b52021-11-18 16:07:01 +0100719 void DepleteTaskQueues() { time_controller_.AdvanceTime(TimeDelta::Zero()); }
Markus Handell8d87c462021-12-16 11:37:16 +0100720 MockFakeEncoder& GetMockFakeEncoder() { return mock_fake_encoder_; }
Markus Handell9a478b52021-11-18 16:07:01 +0100721
Markus Handell818e7fb2021-12-30 13:01:33 +0100722 GlobalSimulatedTimeController* GetTimeController() {
723 return &time_controller_;
724 }
725
Markus Handellb4e96d42021-11-05 12:00:55 +0100726 private:
727 class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
728 public:
729 ~NullEncoderSink() override = default;
730 void OnEncoderConfigurationChanged(
731 std::vector<VideoStream> streams,
732 bool is_svc,
733 VideoEncoderConfig::ContentType content_type,
734 int min_transmit_bitrate_bps) override {}
735 void OnBitrateAllocationUpdated(
736 const VideoBitrateAllocation& allocation) override {}
737 void OnVideoLayersAllocationUpdated(
738 VideoLayersAllocation allocation) override {}
739 Result OnEncodedImage(
740 const EncodedImage& encoded_image,
741 const CodecSpecificInfo* codec_specific_info) override {
742 return Result(EncodedImageCallback::Result::OK);
743 }
744 };
745
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100746 test::ScopedKeyValueConfig field_trials_;
Markus Handellee225432021-11-29 12:35:12 +0100747 GlobalSimulatedTimeController time_controller_{Timestamp::Millis(0)};
748 std::unique_ptr<TaskQueueFactory> task_queue_factory_{
749 time_controller_.CreateTaskQueueFactory()};
750 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_ =
751 std::make_unique<MockableSendStatisticsProxy>(
752 time_controller_.GetClock(),
753 VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 12:52:48 +0100754 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
755 field_trials_);
Markus Handellee225432021-11-29 12:35:12 +0100756 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_ =
757 CreateBuiltinVideoBitrateAllocatorFactory();
758 VideoStreamEncoderSettings encoder_settings_{
759 VideoEncoder::Capabilities(/*loss_notification=*/false)};
Markus Handell8d87c462021-12-16 11:37:16 +0100760 MockFakeEncoder mock_fake_encoder_{time_controller_.GetClock()};
761 test::VideoEncoderProxyFactory encoder_factory_{&mock_fake_encoder_};
Markus Handellb4e96d42021-11-05 12:00:55 +0100762 NullEncoderSink sink_;
763};
764
765class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
766 public:
767 MOCK_METHOD(void, Initialize, (Callback * callback), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100768 MOCK_METHOD(void,
769 SetZeroHertzModeEnabled,
770 (absl::optional<ZeroHertzModeParams>),
771 (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100772 MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
Markus Handellee225432021-11-29 12:35:12 +0100773 MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override));
774 MOCK_METHOD(void, UpdateFrameRate, (), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100775 MOCK_METHOD(void,
776 UpdateLayerQualityConvergence,
777 (int spatial_index, bool converged),
778 (override));
779 MOCK_METHOD(void,
780 UpdateLayerStatus,
781 (int spatial_index, bool enabled),
782 (override));
Markus Handell818e7fb2021-12-30 13:01:33 +0100783 MOCK_METHOD(void, ProcessKeyFrameRequest, (), (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100784};
785
philipel9b058032020-02-10 11:30:00 +0100786class MockEncoderSelector
787 : public VideoEncoderFactory::EncoderSelectorInterface {
788 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200789 MOCK_METHOD(void,
790 OnCurrentEncoder,
791 (const SdpVideoFormat& format),
792 (override));
793 MOCK_METHOD(absl::optional<SdpVideoFormat>,
794 OnAvailableBitrate,
795 (const DataRate& rate),
796 (override));
philipel6daa3042022-04-11 10:48:28 +0200797 MOCK_METHOD(absl::optional<SdpVideoFormat>,
798 OnResolutionChange,
799 (const RenderResolution& resolution),
800 (override));
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200801 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100802};
803
Markus Handell2e0f4f02021-12-21 19:14:58 +0100804class MockVideoSourceInterface : public rtc::VideoSourceInterface<VideoFrame> {
805 public:
806 MOCK_METHOD(void,
807 AddOrUpdateSink,
808 (rtc::VideoSinkInterface<VideoFrame>*,
809 const rtc::VideoSinkWants&),
810 (override));
811 MOCK_METHOD(void,
812 RemoveSink,
813 (rtc::VideoSinkInterface<VideoFrame>*),
814 (override));
815 MOCK_METHOD(void, RequestRefreshFrame, (), (override));
816};
817
perkj803d97f2016-11-01 11:45:46 -0700818} // namespace
819
mflodmancc3d4422017-08-03 08:27:51 -0700820class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700821 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200822 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700823
mflodmancc3d4422017-08-03 08:27:51 -0700824 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700825 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700826 codec_width_(320),
827 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200828 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200829 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200830 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700831 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200832 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700833 video_send_config_,
Jonas Oreland8ca06132022-03-14 12:52:48 +0100834 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
835 field_trials_)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200836 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700837
838 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700839 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700840 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200841 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800842 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200843 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200844 video_send_config_.rtp.payload_name = "FAKE";
845 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700846
Per512ecb32016-09-23 15:52:06 +0200847 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200848 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200849 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
850 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
851 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100852 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700853
Niels Möllerf1338562018-04-26 09:51:47 +0200854 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800855 }
856
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100857 void ConfigureEncoder(
858 VideoEncoderConfig video_encoder_config,
859 VideoStreamEncoder::BitrateAllocationCallbackType
860 allocation_callback_type =
861 VideoStreamEncoder::BitrateAllocationCallbackType::
Erik Språnge4589cb2022-04-06 16:44:30 +0200862 kVideoBitrateAllocationWhenScreenSharing,
863 int num_cores = 1) {
mflodmancc3d4422017-08-03 08:27:51 -0700864 if (video_stream_encoder_)
865 video_stream_encoder_->Stop();
Markus Handell9a478b52021-11-18 16:07:01 +0100866
867 auto encoder_queue = GetTaskQueueFactory()->CreateTaskQueue(
868 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
869 TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
870 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter =
871 FrameCadenceAdapterInterface::Create(time_controller_.GetClock(),
Jonas Oreland8ca06132022-03-14 12:52:48 +0100872 encoder_queue_ptr, field_trials_);
Markus Handell9a478b52021-11-18 16:07:01 +0100873 video_stream_encoder_ = std::make_unique<VideoStreamEncoderUnderTest>(
874 &time_controller_, std::move(cadence_adapter), std::move(encoder_queue),
875 stats_proxy_.get(), video_send_config_.encoder_settings,
Erik Språnge4589cb2022-04-06 16:44:30 +0200876 allocation_callback_type, field_trials_, num_cores);
Asa Persson606d3cb2021-10-04 10:07:11 +0200877 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 08:27:51 -0700878 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700879 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 10:07:11 +0200880 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 08:27:51 -0700881 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200882 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700883 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800884 }
885
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100886 void ResetEncoder(const std::string& payload_name,
887 size_t num_streams,
888 size_t num_temporal_layers,
889 unsigned char num_spatial_layers,
890 bool screenshare,
891 VideoStreamEncoder::BitrateAllocationCallbackType
892 allocation_callback_type =
893 VideoStreamEncoder::BitrateAllocationCallbackType::
Erik Språnge4589cb2022-04-06 16:44:30 +0200894 kVideoBitrateAllocationWhenScreenSharing,
895 int num_cores = 1) {
Niels Möller259a4972018-04-05 15:36:51 +0200896 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800897
898 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200899 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
900 num_streams, &video_encoder_config);
901 for (auto& layer : video_encoder_config.simulcast_layers) {
902 layer.num_temporal_layers = num_temporal_layers;
903 layer.max_framerate = kDefaultFramerate;
904 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100905 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 10:07:11 +0200906 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 07:06:52 -0700907 video_encoder_config.content_type =
908 screenshare ? VideoEncoderConfig::ContentType::kScreen
909 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700910 if (payload_name == "VP9") {
911 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
912 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200913 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700914 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200915 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
916 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700917 }
Erik Språnge4589cb2022-04-06 16:44:30 +0200918 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type,
919 num_cores);
perkj26091b12016-09-01 01:17:40 -0700920 }
921
sprang57c2fff2017-01-16 06:24:02 -0800922 VideoFrame CreateFrame(int64_t ntp_time_ms,
923 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200924 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200925 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200926 destruction_event, codec_width_, codec_height_))
927 .set_ntp_time_ms(ntp_time_ms)
928 .set_timestamp_ms(99)
929 .set_rotation(kVideoRotation_0)
930 .build();
perkj26091b12016-09-01 01:17:40 -0700931 }
932
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100933 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
934 rtc::Event* destruction_event,
935 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200936 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200937 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200938 destruction_event, codec_width_, codec_height_))
939 .set_ntp_time_ms(ntp_time_ms)
940 .set_timestamp_ms(99)
941 .set_rotation(kVideoRotation_0)
942 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
943 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100944 }
945
sprang57c2fff2017-01-16 06:24:02 -0800946 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200947 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
948 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200949 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200950 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200951 .set_ntp_time_ms(ntp_time_ms)
952 .set_timestamp_ms(ntp_time_ms)
953 .set_rotation(kVideoRotation_0)
954 .build();
perkj803d97f2016-11-01 11:45:46 -0700955 }
956
Evan Shrubsole895556e2020-10-05 09:15:13 +0200957 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200958 return VideoFrame::Builder()
959 .set_video_frame_buffer(NV12Buffer::Create(width, height))
960 .set_ntp_time_ms(ntp_time_ms)
961 .set_timestamp_ms(ntp_time_ms)
962 .set_rotation(kVideoRotation_0)
963 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200964 }
965
Noah Richards51db4212019-06-12 06:59:12 -0700966 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
967 rtc::Event* destruction_event,
968 int width,
969 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200970 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200971 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200972 destruction_event, width, height))
973 .set_ntp_time_ms(ntp_time_ms)
974 .set_timestamp_ms(99)
975 .set_rotation(kVideoRotation_0)
976 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700977 }
978
Evan Shrubsole895556e2020-10-05 09:15:13 +0200979 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
980 rtc::Event* destruction_event,
981 int width,
982 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200983 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200984 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200985 destruction_event, width, height))
986 .set_ntp_time_ms(ntp_time_ms)
987 .set_timestamp_ms(99)
988 .set_rotation(kVideoRotation_0)
989 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200990 }
991
Noah Richards51db4212019-06-12 06:59:12 -0700992 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
993 rtc::Event* destruction_event) const {
994 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
995 codec_height_);
996 }
997
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100998 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200999 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001000 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001001
1002 video_source_.IncomingCapturedFrame(
1003 CreateFrame(1, codec_width_, codec_height_));
1004 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +02001005 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001006 }
1007
sprang4847ae62017-06-27 07:06:52 -07001008 void WaitForEncodedFrame(int64_t expected_ntp_time) {
1009 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001010 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001011 }
1012
1013 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
1014 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001015 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001016 return ok;
1017 }
1018
1019 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
1020 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001021 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001022 }
1023
1024 void ExpectDroppedFrame() {
1025 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001026 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001027 }
1028
1029 bool WaitForFrame(int64_t timeout_ms) {
1030 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001031 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001032 return ok;
1033 }
1034
perkj26091b12016-09-01 01:17:40 -07001035 class TestEncoder : public test::FakeEncoder {
1036 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001037 explicit TestEncoder(TimeController* time_controller)
1038 : FakeEncoder(time_controller->GetClock()),
1039 time_controller_(time_controller) {
1040 RTC_DCHECK(time_controller_);
1041 }
perkj26091b12016-09-01 01:17:40 -07001042
Erik Språngaed30702018-11-05 12:57:17 +01001043 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +02001044 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +02001045 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +01001046 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001047 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +01001048 info.scaling_settings = VideoEncoder::ScalingSettings(
1049 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001050 }
1051 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001052 for (int i = 0; i < kMaxSpatialLayers; ++i) {
1053 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001054 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001055 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001056 for (int tid = 0; tid < num_layers; ++tid)
1057 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001058 }
1059 }
Erik Språngaed30702018-11-05 12:57:17 +01001060 }
Sergey Silkin6456e352019-07-08 17:56:40 +02001061
1062 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001063 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001064 info.apply_alignment_to_all_simulcast_layers =
1065 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001066 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001067 if (is_qp_trusted_.has_value()) {
1068 info.is_qp_trusted = is_qp_trusted_;
1069 }
Erik Språngaed30702018-11-05 12:57:17 +01001070 return info;
kthelgason876222f2016-11-29 01:44:11 -08001071 }
1072
Erik Språngb7cb7b52019-02-26 15:52:33 +01001073 int32_t RegisterEncodeCompleteCallback(
1074 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +02001075 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001076 encoded_image_callback_ = callback;
1077 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
1078 }
1079
perkjfa10b552016-10-02 23:45:26 -07001080 void ContinueEncode() { continue_encode_event_.Set(); }
1081
1082 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
1083 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +02001084 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -07001085 EXPECT_EQ(timestamp_, timestamp);
1086 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
1087 }
1088
kthelgason2fc52542017-03-03 00:24:41 -08001089 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +02001090 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -08001091 quality_scaling_ = b;
1092 }
kthelgasonad9010c2017-02-14 00:46:51 -08001093
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001094 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +02001095 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001096 requested_resolution_alignment_ = requested_resolution_alignment;
1097 }
1098
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001099 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
1100 MutexLock lock(&local_mutex_);
1101 apply_alignment_to_all_simulcast_layers_ = b;
1102 }
1103
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001104 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +02001105 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001106 is_hardware_accelerated_ = is_hardware_accelerated;
1107 }
1108
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001109 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
1110 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +02001111 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001112 temporal_layers_supported_[spatial_idx] = supported;
1113 }
1114
Sergey Silkin6456e352019-07-08 17:56:40 +02001115 void SetResolutionBitrateLimits(
1116 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +02001117 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +02001118 resolution_bitrate_limits_ = thresholds;
1119 }
1120
sprangfe627f32017-03-29 08:24:59 -07001121 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +02001122 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -07001123 force_init_encode_failed_ = force_failure;
1124 }
1125
Niels Möller6bb5ab92019-01-11 11:11:10 +01001126 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +02001127 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001128 rate_factor_ = rate_factor;
1129 }
1130
Erik Språngd7329ca2019-02-21 21:19:53 +01001131 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +02001132 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001133 return last_framerate_;
1134 }
1135
Erik Språngd7329ca2019-02-21 21:19:53 +01001136 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +02001137 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001138 return last_update_rect_;
1139 }
1140
Niels Möller87e2d782019-03-07 10:18:23 +01001141 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +02001142 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001143 return last_frame_types_;
1144 }
1145
1146 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +01001147 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +01001148 keyframe ? VideoFrameType::kVideoFrameKey
1149 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01001150 {
Markus Handella3765182020-07-08 13:13:32 +02001151 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001152 last_frame_types_ = frame_type;
1153 }
Niels Möllerb859b322019-03-07 12:40:01 +01001154 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +01001155 }
1156
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001157 void InjectEncodedImage(const EncodedImage& image,
1158 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +02001159 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001160 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001161 }
1162
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001163 void SetEncodedImageData(
1164 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +02001165 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001166 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001167 }
1168
Erik Språngd7329ca2019-02-21 21:19:53 +01001169 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +02001170 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001171 expect_null_frame_ = true;
1172 }
1173
Erik Språng5056af02019-09-02 15:53:11 +02001174 absl::optional<VideoEncoder::RateControlParameters>
1175 GetAndResetLastRateControlSettings() {
1176 auto settings = last_rate_control_settings_;
1177 last_rate_control_settings_.reset();
1178 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +01001179 }
1180
Henrik Boström56db9ff2021-03-24 09:06:45 +01001181 int GetLastInputWidth() const {
1182 MutexLock lock(&local_mutex_);
1183 return last_input_width_;
1184 }
1185
1186 int GetLastInputHeight() const {
1187 MutexLock lock(&local_mutex_);
1188 return last_input_height_;
1189 }
1190
Evan Shrubsole895556e2020-10-05 09:15:13 +02001191 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1192 MutexLock lock(&local_mutex_);
1193 return last_input_pixel_format_;
1194 }
1195
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001196 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001197 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001198 return num_set_rates_;
1199 }
1200
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001201 void SetPreferredPixelFormats(
1202 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1203 pixel_formats) {
1204 MutexLock lock(&local_mutex_);
1205 preferred_pixel_formats_ = std::move(pixel_formats);
1206 }
1207
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001208 void SetIsQpTrusted(absl::optional<bool> trusted) {
1209 MutexLock lock(&local_mutex_);
1210 is_qp_trusted_ = trusted;
1211 }
1212
Erik Språnge4589cb2022-04-06 16:44:30 +02001213 VideoCodecComplexity LastEncoderComplexity() {
1214 MutexLock lock(&local_mutex_);
1215 return last_encoder_complexity_;
1216 }
1217
perkjfa10b552016-10-02 23:45:26 -07001218 private:
perkj26091b12016-09-01 01:17:40 -07001219 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001220 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001221 {
Markus Handella3765182020-07-08 13:13:32 +02001222 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001223 if (expect_null_frame_) {
1224 EXPECT_EQ(input_image.timestamp(), 0u);
1225 EXPECT_EQ(input_image.width(), 1);
1226 last_frame_types_ = *frame_types;
1227 expect_null_frame_ = false;
1228 } else {
1229 EXPECT_GT(input_image.timestamp(), timestamp_);
1230 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1231 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1232 }
perkj26091b12016-09-01 01:17:40 -07001233
1234 timestamp_ = input_image.timestamp();
1235 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001236 last_input_width_ = input_image.width();
1237 last_input_height_ = input_image.height();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001238 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001239 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001240 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001241 }
Niels Möllerb859b322019-03-07 12:40:01 +01001242 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001243 return result;
1244 }
1245
Niels Möller08ae7ce2020-09-23 15:58:12 +02001246 CodecSpecificInfo EncodeHook(
1247 EncodedImage& encoded_image,
1248 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001249 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001250 {
1251 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001252 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001253 }
1254 MutexLock lock(&local_mutex_);
1255 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001256 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001257 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001258 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001259 }
1260
sprangfe627f32017-03-29 08:24:59 -07001261 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001262 const Settings& settings) override {
1263 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001264
Markus Handella3765182020-07-08 13:13:32 +02001265 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001266 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001267
Erik Språng82fad3d2018-03-21 09:57:23 +01001268 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001269 // Simulate setting up temporal layers, in order to validate the life
1270 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001271 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001272 frame_buffer_controller_ =
1273 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001274 }
Erik Språnge4589cb2022-04-06 16:44:30 +02001275
1276 last_encoder_complexity_ = config->GetVideoEncoderComplexity();
1277
Erik Språngb7cb7b52019-02-26 15:52:33 +01001278 if (force_init_encode_failed_) {
1279 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001280 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001281 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001282
Erik Språngb7cb7b52019-02-26 15:52:33 +01001283 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001284 return res;
1285 }
1286
Erik Språngb7cb7b52019-02-26 15:52:33 +01001287 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001288 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001289 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1290 initialized_ = EncoderState::kUninitialized;
1291 return FakeEncoder::Release();
1292 }
1293
Erik Språng16cb8f52019-04-12 13:59:09 +02001294 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001295 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001296 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001297 VideoBitrateAllocation adjusted_rate_allocation;
1298 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1299 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001300 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001301 adjusted_rate_allocation.SetBitrate(
1302 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001303 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001304 rate_factor_));
1305 }
1306 }
1307 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001308 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001309 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001310 RateControlParameters adjusted_paramters = parameters;
1311 adjusted_paramters.bitrate = adjusted_rate_allocation;
1312 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001313 }
1314
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001315 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001316 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001317 enum class EncoderState {
1318 kUninitialized,
1319 kInitializationFailed,
1320 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001321 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
perkj26091b12016-09-01 01:17:40 -07001322 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001323 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1324 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1325 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1326 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1327 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1328 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001329 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1330 false;
Markus Handella3765182020-07-08 13:13:32 +02001331 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001332 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1333 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001334 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001335 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001336 absl::optional<bool>
1337 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001338 local_mutex_);
1339 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1340 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1341 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001342 absl::optional<VideoEncoder::RateControlParameters>
1343 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001344 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1345 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001346 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001347 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001348 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1349 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001350 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 17:56:40 +02001351 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001352 RTC_GUARDED_BY(local_mutex_);
1353 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001354 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1355 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001356 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1357 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001358 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
Erik Språnge4589cb2022-04-06 16:44:30 +02001359 VideoCodecComplexity last_encoder_complexity_ RTC_GUARDED_BY(local_mutex_){
1360 VideoCodecComplexity::kComplexityNormal};
perkj26091b12016-09-01 01:17:40 -07001361 };
1362
mflodmancc3d4422017-08-03 08:27:51 -07001363 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001364 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001365 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1366 : time_controller_(time_controller), test_encoder_(test_encoder) {
1367 RTC_DCHECK(time_controller_);
1368 }
perkj26091b12016-09-01 01:17:40 -07001369
perkj26091b12016-09-01 01:17:40 -07001370 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001371 EXPECT_TRUE(
1372 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1373 }
1374
1375 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1376 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001377 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001378 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001379 return false;
perkj26091b12016-09-01 01:17:40 -07001380 {
Markus Handella3765182020-07-08 13:13:32 +02001381 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001382 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001383 }
1384 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001385 return true;
perkj26091b12016-09-01 01:17:40 -07001386 }
1387
sprangb1ca0732017-02-01 08:38:12 -08001388 void WaitForEncodedFrame(uint32_t expected_width,
1389 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001390 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001391 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001392 }
1393
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001394 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001395 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001396 uint32_t width = 0;
1397 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001398 {
Markus Handella3765182020-07-08 13:13:32 +02001399 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001400 width = last_width_;
1401 height = last_height_;
1402 }
1403 EXPECT_EQ(expected_height, height);
1404 EXPECT_EQ(expected_width, width);
1405 }
1406
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001407 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1408 VideoRotation rotation;
1409 {
Markus Handella3765182020-07-08 13:13:32 +02001410 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001411 rotation = last_rotation_;
1412 }
1413 EXPECT_EQ(expected_rotation, rotation);
1414 }
1415
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001416 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001417
sprangc5d62e22017-04-02 23:53:04 -07001418 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001419 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +01001420 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001421 bool ret = encoded_frame_event_.Wait(timeout_ms);
Markus Handell28c71802021-11-08 10:11:55 +01001422 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001423 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001424 }
1425
perkj26091b12016-09-01 01:17:40 -07001426 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001427 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001428 expect_frames_ = false;
1429 }
1430
asaperssonfab67072017-04-04 05:51:49 -07001431 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001432 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001433 return number_of_reconfigurations_;
1434 }
1435
asaperssonfab67072017-04-04 05:51:49 -07001436 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001437 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001438 return min_transmit_bitrate_bps_;
1439 }
1440
Erik Språngd7329ca2019-02-21 21:19:53 +01001441 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001442 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001443 num_expected_layers_ = num_layers;
1444 }
1445
Erik Språngb7cb7b52019-02-26 15:52:33 +01001446 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001447 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001448 return last_capture_time_ms_;
1449 }
1450
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001451 const EncodedImage& GetLastEncodedImage() {
1452 MutexLock lock(&mutex_);
1453 return last_encoded_image_;
1454 }
1455
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001456 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001457 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001458 return std::move(last_encoded_image_data_);
1459 }
1460
Per Kjellanderdcef6412020-10-07 15:09:05 +02001461 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1462 MutexLock lock(&mutex_);
1463 return last_bitrate_allocation_;
1464 }
1465
1466 int number_of_bitrate_allocations() const {
1467 MutexLock lock(&mutex_);
1468 return number_of_bitrate_allocations_;
1469 }
1470
Per Kjellandera9434842020-10-15 17:53:22 +02001471 VideoLayersAllocation GetLastVideoLayersAllocation() {
1472 MutexLock lock(&mutex_);
1473 return last_layers_allocation_;
1474 }
1475
1476 int number_of_layers_allocations() const {
1477 MutexLock lock(&mutex_);
1478 return number_of_layers_allocations_;
1479 }
1480
perkj26091b12016-09-01 01:17:40 -07001481 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001482 Result OnEncodedImage(
1483 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001484 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001485 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001486 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001487 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001488 last_encoded_image_data_ = std::vector<uint8_t>(
1489 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001490 uint32_t timestamp = encoded_image.Timestamp();
1491 if (last_timestamp_ != timestamp) {
1492 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001493 last_width_ = encoded_image._encodedWidth;
1494 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001495 } else {
1496 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001497 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1498 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001499 }
1500 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001501 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001502 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001503 if (num_received_layers_ == num_expected_layers_) {
1504 encoded_frame_event_.Set();
1505 }
sprangb1ca0732017-02-01 08:38:12 -08001506 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001507 }
1508
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001509 void OnEncoderConfigurationChanged(
1510 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001511 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001512 VideoEncoderConfig::ContentType content_type,
1513 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001514 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001515 ++number_of_reconfigurations_;
1516 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1517 }
1518
Per Kjellanderdcef6412020-10-07 15:09:05 +02001519 void OnBitrateAllocationUpdated(
1520 const VideoBitrateAllocation& allocation) override {
1521 MutexLock lock(&mutex_);
1522 ++number_of_bitrate_allocations_;
1523 last_bitrate_allocation_ = allocation;
1524 }
1525
Per Kjellandera9434842020-10-15 17:53:22 +02001526 void OnVideoLayersAllocationUpdated(
1527 VideoLayersAllocation allocation) override {
1528 MutexLock lock(&mutex_);
1529 ++number_of_layers_allocations_;
1530 last_layers_allocation_ = allocation;
1531 rtc::StringBuilder log;
1532 for (const auto& layer : allocation.active_spatial_layers) {
1533 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1534 << "[";
1535 for (const auto target_bitrate :
1536 layer.target_bitrate_per_temporal_layer) {
1537 log << target_bitrate.kbps() << ",";
1538 }
1539 log << "]";
1540 }
Harald Alvestrand97597c02021-11-04 12:01:23 +00001541 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 17:53:22 +02001542 }
1543
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001544 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001545 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001546 TestEncoder* test_encoder_;
1547 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001548 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001549 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001550 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001551 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001552 uint32_t last_height_ = 0;
1553 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001554 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001555 size_t num_expected_layers_ = 1;
1556 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001557 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001558 int number_of_reconfigurations_ = 0;
1559 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001560 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1561 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001562 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1563 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001564 };
1565
Sergey Silkin5ee69672019-07-02 14:18:34 +02001566 class VideoBitrateAllocatorProxyFactory
1567 : public VideoBitrateAllocatorFactory {
1568 public:
1569 VideoBitrateAllocatorProxyFactory()
1570 : bitrate_allocator_factory_(
1571 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1572
1573 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1574 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001575 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001576 codec_config_ = codec;
1577 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1578 }
1579
1580 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001581 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001582 return codec_config_;
1583 }
1584
1585 private:
1586 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1587
Markus Handella3765182020-07-08 13:13:32 +02001588 mutable Mutex mutex_;
1589 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001590 };
1591
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001592 Clock* clock() { return time_controller_.GetClock(); }
1593 void AdvanceTime(TimeDelta duration) {
1594 time_controller_.AdvanceTime(duration);
1595 }
1596
1597 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1598
1599 protected:
1600 virtual TaskQueueFactory* GetTaskQueueFactory() {
1601 return time_controller_.GetTaskQueueFactory();
1602 }
1603
Jonas Orelandc7f691a2022-03-09 15:12:07 +01001604 test::ScopedKeyValueConfig field_trials_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001605 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001606 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001607 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001608 int codec_width_;
1609 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001610 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001611 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001612 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001613 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001614 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001615 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001616 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001617 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001618};
1619
mflodmancc3d4422017-08-03 08:27:51 -07001620TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001621 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001622 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001623 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001624 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001625 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001626 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001627 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001628}
1629
mflodmancc3d4422017-08-03 08:27:51 -07001630TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001631 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001632 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001633 // The encoder will cache up to one frame for a short duration. Adding two
1634 // frames means that the first frame will be dropped and the second frame will
1635 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001636 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001637 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001638 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Markus Handell28c71802021-11-08 10:11:55 +01001639 AdvanceTime(TimeDelta::Zero());
perkja49cbd32016-09-16 07:53:41 -07001640 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001641
Henrik Boström381d1092020-05-12 18:49:07 +02001642 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001643 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001644
Sebastian Janssona3177052018-04-10 13:05:49 +02001645 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001646 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001647 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1648
1649 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001650 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001651}
1652
mflodmancc3d4422017-08-03 08:27:51 -07001653TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001654 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001655 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001656 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001657 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001658
Henrik Boström381d1092020-05-12 18:49:07 +02001659 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001660 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1661
Sebastian Janssona3177052018-04-10 13:05:49 +02001662 // The encoder will cache up to one frame for a short duration. Adding two
1663 // frames means that the first frame will be dropped and the second frame will
1664 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001665 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001666 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001667
Henrik Boström381d1092020-05-12 18:49:07 +02001668 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001669 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001670 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001671 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1672 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001673 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001674}
1675
mflodmancc3d4422017-08-03 08:27:51 -07001676TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001677 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001678 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001679 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001680 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001681
1682 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001683 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001684
perkja49cbd32016-09-16 07:53:41 -07001685 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001686 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001687 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001688}
1689
mflodmancc3d4422017-08-03 08:27:51 -07001690TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001691 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001692 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001693
perkja49cbd32016-09-16 07:53:41 -07001694 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001695 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001696
mflodmancc3d4422017-08-03 08:27:51 -07001697 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001698 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001699 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001700 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1701 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001702}
1703
Markus Handell9a478b52021-11-18 16:07:01 +01001704TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
1705 test::FrameForwarder source;
1706 video_stream_encoder_->SetSource(&source,
1707 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02001708 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001709 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001710
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001711 int dropped_count = 0;
1712 stats_proxy_->SetDroppedFrameCallback(
1713 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1714 ++dropped_count;
1715 });
1716
Markus Handell9a478b52021-11-18 16:07:01 +01001717 source.IncomingCapturedFrame(CreateFrame(1, nullptr));
1718 source.IncomingCapturedFrame(CreateFrame(2, nullptr));
1719 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001720 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001721 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001722}
1723
Henrik Boström56db9ff2021-03-24 09:06:45 +01001724TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001725 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001726 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001727
1728 rtc::Event frame_destroyed_event;
1729 video_source_.IncomingCapturedFrame(
1730 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001731 WaitForEncodedFrame(1);
1732 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1733 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001734 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1735 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001736 video_stream_encoder_->Stop();
1737}
1738
Henrik Boström56db9ff2021-03-24 09:06:45 +01001739TEST_F(VideoStreamEncoderTest,
1740 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001741 // Use the cropping factory.
1742 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001743 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001744 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1745 kMaxPayloadLength);
1746 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1747
1748 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001749 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001750 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001751 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1752 WaitForEncodedFrame(1);
1753 // The encoder will have been configured once.
1754 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001755 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1756 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 06:59:12 -07001757
1758 // Now send in a fake frame that needs to be cropped as the width/height
1759 // aren't divisible by 4 (see CreateEncoderStreams above).
1760 rtc::Event frame_destroyed_event;
1761 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1762 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001763 WaitForEncodedFrame(2);
1764 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1765 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001766 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1767 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001768 video_stream_encoder_->Stop();
1769}
1770
Evan Shrubsole895556e2020-10-05 09:15:13 +02001771TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1772 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001773 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001774
1775 video_source_.IncomingCapturedFrame(
1776 CreateNV12Frame(1, codec_width_, codec_height_));
1777 WaitForEncodedFrame(1);
1778 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1779 fake_encoder_.GetLastInputPixelFormat());
1780 video_stream_encoder_->Stop();
1781}
1782
Henrik Boström56db9ff2021-03-24 09:06:45 +01001783TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001784 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001785 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001786
1787 fake_encoder_.SetPreferredPixelFormats({});
1788
1789 rtc::Event frame_destroyed_event;
1790 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1791 1, &frame_destroyed_event, codec_width_, codec_height_));
1792 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001793 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001794 fake_encoder_.GetLastInputPixelFormat());
1795 video_stream_encoder_->Stop();
1796}
1797
Henrik Boström56db9ff2021-03-24 09:06:45 +01001798TEST_F(VideoStreamEncoderTest,
1799 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001800 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001801 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001802
1803 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1804
1805 rtc::Event frame_destroyed_event;
1806 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1807 1, &frame_destroyed_event, codec_width_, codec_height_));
1808 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001809 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001810 fake_encoder_.GetLastInputPixelFormat());
1811 video_stream_encoder_->Stop();
1812}
1813
Henrik Boström56db9ff2021-03-24 09:06:45 +01001814TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001815 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001816 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001817
1818 // Fake NV12 native frame does not allow mapping to I444.
1819 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1820
1821 rtc::Event frame_destroyed_event;
1822 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1823 1, &frame_destroyed_event, codec_width_, codec_height_));
1824 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001825 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001826 fake_encoder_.GetLastInputPixelFormat());
1827 video_stream_encoder_->Stop();
1828}
1829
Henrik Boström56db9ff2021-03-24 09:06:45 +01001830TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001831 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001832 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001833
1834 rtc::Event frame_destroyed_event;
1835 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1836 1, &frame_destroyed_event, codec_width_, codec_height_));
1837 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001838 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001839 fake_encoder_.GetLastInputPixelFormat());
1840 video_stream_encoder_->Stop();
1841}
1842
Ying Wang9b881ab2020-02-07 14:29:32 +01001843TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001844 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001845 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001846 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1847 WaitForEncodedFrame(1);
1848
Henrik Boström381d1092020-05-12 18:49:07 +02001849 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001850 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001851 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1852 // frames. Adding two frames means that the first frame will be dropped and
1853 // the second frame will be sent to the encoder.
1854 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1855 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1856 WaitForEncodedFrame(3);
1857 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1858 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1859 WaitForEncodedFrame(5);
1860 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1861 video_stream_encoder_->Stop();
1862}
1863
mflodmancc3d4422017-08-03 08:27:51 -07001864TEST_F(VideoStreamEncoderTest,
1865 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001866 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001867 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001868 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001869
1870 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001871 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001872 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001873 // The encoder will have been configured once when the first frame is
1874 // received.
1875 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001876
1877 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001878 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001879 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001880 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001881 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001882
1883 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001884 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001885 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001886 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001887 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001888
mflodmancc3d4422017-08-03 08:27:51 -07001889 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001890}
1891
mflodmancc3d4422017-08-03 08:27:51 -07001892TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001893 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001894 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001895
1896 // Capture a frame and wait for it to synchronize with the encoder thread.
1897 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001898 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001899 // The encoder will have been configured once.
1900 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001901 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1902 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-02 23:45:26 -07001903
1904 codec_width_ *= 2;
1905 codec_height_ *= 2;
1906 // Capture a frame with a higher resolution and wait for it to synchronize
1907 // with the encoder thread.
1908 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001909 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 10:07:11 +02001910 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1911 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 21:37:57 +01001912 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001913
mflodmancc3d4422017-08-03 08:27:51 -07001914 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001915}
1916
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001917TEST_F(VideoStreamEncoderTest,
1918 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001919 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001920 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001921
1922 // Capture a frame and wait for it to synchronize with the encoder thread.
1923 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1924 WaitForEncodedFrame(1);
1925
1926 VideoEncoderConfig video_encoder_config;
1927 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1928 // Changing the max payload data length recreates encoder.
1929 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1930 kMaxPayloadLength / 2);
1931
1932 // Capture a frame and wait for it to synchronize with the encoder thread.
1933 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1934 WaitForEncodedFrame(2);
1935 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1936
1937 video_stream_encoder_->Stop();
1938}
1939
Sergey Silkin5ee69672019-07-02 14:18:34 +02001940TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001941 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001942 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001943
1944 VideoEncoderConfig video_encoder_config;
1945 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02001946 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1947 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001948 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1949 kMaxPayloadLength);
1950
1951 // Capture a frame and wait for it to synchronize with the encoder thread.
1952 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1953 WaitForEncodedFrame(1);
1954 // The encoder will have been configured once when the first frame is
1955 // received.
1956 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001957 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001958 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001959 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001960 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1961
Sergey Silkin6456e352019-07-08 17:56:40 +02001962 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1963 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 10:07:11 +02001964 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1965 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001966 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1967 kMaxPayloadLength);
1968
1969 // Capture a frame and wait for it to synchronize with the encoder thread.
1970 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1971 WaitForEncodedFrame(2);
1972 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1973 // Bitrate limits have changed - rate allocator should be reconfigured,
1974 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 10:07:11 +02001975 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001976 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001977 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001978 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001979 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001980
1981 video_stream_encoder_->Stop();
1982}
1983
Sergey Silkin6456e352019-07-08 17:56:40 +02001984TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001985 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001986 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001987 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001988
Sergey Silkincd02eba2020-01-20 14:48:40 +01001989 const uint32_t kMinEncBitrateKbps = 100;
1990 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001991 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001992 /*frame_size_pixels=*/codec_width_ * codec_height_,
1993 /*min_start_bitrate_bps=*/0,
1994 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1995 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001996 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1997
Sergey Silkincd02eba2020-01-20 14:48:40 +01001998 VideoEncoderConfig video_encoder_config;
1999 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2000 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
2001 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2002 (kMinEncBitrateKbps + 1) * 1000;
2003 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2004 kMaxPayloadLength);
2005
2006 // When both encoder and app provide bitrate limits, the intersection of
2007 // provided sets should be used.
2008 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2009 WaitForEncodedFrame(1);
2010 EXPECT_EQ(kMaxEncBitrateKbps,
2011 bitrate_allocator_factory_.codec_config().maxBitrate);
2012 EXPECT_EQ(kMinEncBitrateKbps + 1,
2013 bitrate_allocator_factory_.codec_config().minBitrate);
2014
2015 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
2016 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2017 (kMinEncBitrateKbps - 1) * 1000;
2018 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2019 kMaxPayloadLength);
2020 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002021 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002022 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002023 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002024 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002025 bitrate_allocator_factory_.codec_config().minBitrate);
2026
Sergey Silkincd02eba2020-01-20 14:48:40 +01002027 video_stream_encoder_->Stop();
2028}
2029
2030TEST_F(VideoStreamEncoderTest,
2031 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02002032 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002033 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002034
2035 const uint32_t kMinAppBitrateKbps = 100;
2036 const uint32_t kMaxAppBitrateKbps = 200;
2037 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
2038 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
2039 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2040 /*frame_size_pixels=*/codec_width_ * codec_height_,
2041 /*min_start_bitrate_bps=*/0,
2042 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
2043 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
2044 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2045
2046 VideoEncoderConfig video_encoder_config;
2047 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2048 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
2049 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2050 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002051 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2052 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002053
Sergey Silkincd02eba2020-01-20 14:48:40 +01002054 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2055 WaitForEncodedFrame(1);
2056 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002057 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002058 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002059 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02002060
2061 video_stream_encoder_->Stop();
2062}
2063
2064TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002065 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02002066 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002067 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02002068
2069 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002070 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002071 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002072 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002073 fake_encoder_.SetResolutionBitrateLimits(
2074 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
2075
2076 VideoEncoderConfig video_encoder_config;
2077 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2078 video_encoder_config.max_bitrate_bps = 0;
2079 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2080 kMaxPayloadLength);
2081
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002082 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002083 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2084 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002085 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2086 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002087 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2088 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2089
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002090 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002091 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2092 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002093 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2094 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002095 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2096 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2097
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002098 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02002099 // encoder for 360p should be used.
2100 video_source_.IncomingCapturedFrame(
2101 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2102 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002103 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2104 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002105 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2106 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2107
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002108 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02002109 // ignored.
2110 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2111 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002112 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2113 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002114 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2115 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002116 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2117 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002118 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2119 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2120
2121 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2122 // for 270p should be used.
2123 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2124 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002125 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2126 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002127 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2128 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2129
2130 video_stream_encoder_->Stop();
2131}
2132
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002133TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002134 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002135 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002136
2137 VideoEncoderConfig video_encoder_config;
2138 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2139 video_encoder_config.max_bitrate_bps = 0;
2140 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2141 kMaxPayloadLength);
2142
2143 // Encode 720p frame to get the default encoder target bitrate.
2144 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2145 WaitForEncodedFrame(1);
2146 const uint32_t kDefaultTargetBitrateFor720pKbps =
2147 bitrate_allocator_factory_.codec_config()
2148 .simulcastStream[0]
2149 .targetBitrate;
2150
2151 // Set the max recommended encoder bitrate to something lower than the default
2152 // target bitrate.
2153 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2154 1280 * 720, 10 * 1000, 10 * 1000,
2155 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2156 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2157
2158 // Change resolution to trigger encoder reinitialization.
2159 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2160 WaitForEncodedFrame(2);
2161 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2162 WaitForEncodedFrame(3);
2163
2164 // Ensure the target bitrate is capped by the max bitrate.
2165 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2166 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2167 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2168 .simulcastStream[0]
2169 .targetBitrate *
2170 1000,
2171 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2172
2173 video_stream_encoder_->Stop();
2174}
2175
Åsa Perssona7e34d32021-01-20 15:36:13 +01002176TEST_F(VideoStreamEncoderTest,
2177 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2178 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2179 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2180 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2181 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2182 fake_encoder_.SetResolutionBitrateLimits(
2183 {kEncoderLimits270p, kEncoderLimits360p});
2184
2185 // Two streams, highest stream active.
2186 VideoEncoderConfig config;
2187 const int kNumStreams = 2;
2188 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2189 config.max_bitrate_bps = 0;
2190 config.simulcast_layers[0].active = false;
2191 config.simulcast_layers[1].active = true;
2192 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002193 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002194 "VP8", /*max qp*/ 56, /*screencast*/ false,
2195 /*screenshare enabled*/ false);
2196 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2197
2198 // The encoder bitrate limits for 270p should be used.
2199 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2200 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002201 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002202 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002203 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002204 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002205 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002206
2207 // The encoder bitrate limits for 360p should be used.
2208 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2209 EXPECT_FALSE(WaitForFrame(1000));
2210 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002211 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002212 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002213 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002214
2215 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2216 video_source_.IncomingCapturedFrame(
2217 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2218 EXPECT_FALSE(WaitForFrame(1000));
2219 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002220 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002221 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002222 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002223
2224 // Resolution higher than 360p. Encoder limits should be ignored.
2225 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2226 EXPECT_FALSE(WaitForFrame(1000));
2227 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002228 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002229 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002230 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002231 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002232 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002233 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002234 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002235
2236 // Resolution lower than 270p. The encoder limits for 270p should be used.
2237 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2238 EXPECT_FALSE(WaitForFrame(1000));
2239 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002240 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002241 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002242 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002243
2244 video_stream_encoder_->Stop();
2245}
2246
2247TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002248 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2249 // Two streams, highest stream active.
2250 VideoEncoderConfig config;
2251 const int kNumStreams = 2;
2252 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2253 config.max_bitrate_bps = 0;
2254 config.simulcast_layers[0].active = false;
2255 config.simulcast_layers[1].active = true;
2256 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002257 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002258 "VP8", /*max qp*/ 56, /*screencast*/ false,
2259 /*screenshare enabled*/ false);
2260 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2261
2262 // Default bitrate limits for 270p should be used.
2263 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2264 kDefaultLimits270p =
2265 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002266 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002267 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2268 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002269 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 10:39:51 +01002270 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002271 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002272 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002273 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002274
2275 // Default bitrate limits for 360p should be used.
2276 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2277 kDefaultLimits360p =
2278 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002279 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002280 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2281 EXPECT_FALSE(WaitForFrame(1000));
2282 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002283 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002284 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002285 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002286
2287 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2288 video_source_.IncomingCapturedFrame(
2289 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2290 EXPECT_FALSE(WaitForFrame(1000));
2291 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002292 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002293 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002294 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002295
2296 // Default bitrate limits for 540p should be used.
2297 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2298 kDefaultLimits540p =
2299 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002300 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002301 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2302 EXPECT_FALSE(WaitForFrame(1000));
2303 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002304 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002305 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002306 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002307
2308 video_stream_encoder_->Stop();
2309}
2310
2311TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002312 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2313 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2314 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2315 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2316 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2317 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2318 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2319 fake_encoder_.SetResolutionBitrateLimits(
2320 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2321
2322 // Three streams, middle stream active.
2323 VideoEncoderConfig config;
2324 const int kNumStreams = 3;
2325 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2326 config.simulcast_layers[0].active = false;
2327 config.simulcast_layers[1].active = true;
2328 config.simulcast_layers[2].active = false;
2329 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002330 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002331 "VP8", /*max qp*/ 56, /*screencast*/ false,
2332 /*screenshare enabled*/ false);
2333 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2334
2335 // The encoder bitrate limits for 360p should be used.
2336 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2337 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002338 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002339 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002340 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002341 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002342 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002343
2344 // The encoder bitrate limits for 270p should be used.
2345 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2346 EXPECT_FALSE(WaitForFrame(1000));
2347 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002348 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002349 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002350 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002351
2352 video_stream_encoder_->Stop();
2353}
2354
2355TEST_F(VideoStreamEncoderTest,
2356 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2357 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2358 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2359 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2360 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2361 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2362 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2363 fake_encoder_.SetResolutionBitrateLimits(
2364 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2365
2366 // Three streams, lowest stream active.
2367 VideoEncoderConfig config;
2368 const int kNumStreams = 3;
2369 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2370 config.simulcast_layers[0].active = true;
2371 config.simulcast_layers[1].active = false;
2372 config.simulcast_layers[2].active = false;
2373 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002374 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002375 "VP8", /*max qp*/ 56, /*screencast*/ false,
2376 /*screenshare enabled*/ false);
2377 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2378
2379 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2380 // on lowest stream, limits for 270p should not be used
2381 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2382 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002383 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002384 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002385 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002386 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002387 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002388
2389 video_stream_encoder_->Stop();
2390}
2391
2392TEST_F(VideoStreamEncoderTest,
2393 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2394 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2395 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2396 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2397 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2398 fake_encoder_.SetResolutionBitrateLimits(
2399 {kEncoderLimits270p, kEncoderLimits360p});
2400 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2401
2402 // Two streams, highest stream active.
2403 VideoEncoderConfig config;
2404 const int kNumStreams = 2;
2405 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2406 config.simulcast_layers[0].active = false;
2407 config.simulcast_layers[1].active = true;
2408 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2409 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002410 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002411 "VP8", /*max qp*/ 56, /*screencast*/ false,
2412 /*screenshare enabled*/ false);
2413 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2414
2415 // The encoder bitrate limits for 270p should be used.
2416 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2417 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002418 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002419 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002420 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002421 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002422 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002423
2424 // The max configured bitrate is less than the encoder limit for 360p.
2425 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2426 EXPECT_FALSE(WaitForFrame(1000));
2427 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002428 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002429 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002430 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002431
2432 video_stream_encoder_->Stop();
2433}
2434
mflodmancc3d4422017-08-03 08:27:51 -07002435TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002436 EXPECT_TRUE(video_source_.has_sinks());
2437 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002438 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002439 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002440 EXPECT_FALSE(video_source_.has_sinks());
2441 EXPECT_TRUE(new_video_source.has_sinks());
2442
mflodmancc3d4422017-08-03 08:27:51 -07002443 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002444}
2445
mflodmancc3d4422017-08-03 08:27:51 -07002446TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002447 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002448 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002449 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002450 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002451}
2452
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002453class ResolutionAlignmentTest
2454 : public VideoStreamEncoderTest,
2455 public ::testing::WithParamInterface<
2456 ::testing::tuple<int, std::vector<double>>> {
2457 public:
2458 ResolutionAlignmentTest()
2459 : requested_alignment_(::testing::get<0>(GetParam())),
2460 scale_factors_(::testing::get<1>(GetParam())) {}
2461
2462 protected:
2463 const int requested_alignment_;
2464 const std::vector<double> scale_factors_;
2465};
2466
2467INSTANTIATE_TEST_SUITE_P(
2468 AlignmentAndScaleFactors,
2469 ResolutionAlignmentTest,
2470 ::testing::Combine(
2471 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2472 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2473 std::vector<double>{-1.0, -1.0},
2474 std::vector<double>{-1.0, -1.0, -1.0},
2475 std::vector<double>{4.0, 2.0, 1.0},
2476 std::vector<double>{9999.0, -1.0, 1.0},
2477 std::vector<double>{3.99, 2.01, 1.0},
2478 std::vector<double>{4.9, 1.7, 1.25},
2479 std::vector<double>{10.0, 4.0, 3.0},
2480 std::vector<double>{1.75, 3.5},
2481 std::vector<double>{1.5, 2.5},
2482 std::vector<double>{1.3, 1.0})));
2483
2484TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2485 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002486 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002487 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2488 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2489
2490 // Fill config with the scaling factor by which to reduce encoding size.
2491 const int num_streams = scale_factors_.size();
2492 VideoEncoderConfig config;
2493 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2494 for (int i = 0; i < num_streams; ++i) {
2495 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2496 }
2497 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002498 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002499 "VP8", /*max qp*/ 56, /*screencast*/ false,
2500 /*screenshare enabled*/ false);
2501 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2502
Henrik Boström381d1092020-05-12 18:49:07 +02002503 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002504 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2505 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002506 // Wait for all layers before triggering event.
2507 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002508
2509 // On the 1st frame, we should have initialized the encoder and
2510 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002511 int64_t timestamp_ms = kFrameIntervalMs;
2512 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2513 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002514 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002515
2516 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2517 // (It's up the to the encoder to potentially drop the previous frame,
2518 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002519 timestamp_ms += kFrameIntervalMs;
2520 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2521 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002522 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002523
Asa Persson606d3cb2021-10-04 10:07:11 +02002524 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002525 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2526 // Frame size should be a multiple of the requested alignment.
2527 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2528 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2529 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2530 // Aspect ratio should match.
2531 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2532 codec.height * codec.simulcastStream[i].width);
2533 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002534
2535 video_stream_encoder_->Stop();
2536}
2537
Jonathan Yubc771b72017-12-08 17:04:29 -08002538TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2539 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002540 const int kWidth = 1280;
2541 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002542
2543 // We rely on the automatic resolution adaptation, but we handle framerate
2544 // adaptation manually by mocking the stats proxy.
2545 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002546
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002547 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002548 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002549 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002550 video_stream_encoder_->SetSource(&video_source_,
2551 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002552 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002553 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002554 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002555 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2556
Jonathan Yubc771b72017-12-08 17:04:29 -08002557 // Adapt down as far as possible.
2558 rtc::VideoSinkWants last_wants;
2559 int64_t t = 1;
2560 int loop_count = 0;
2561 do {
2562 ++loop_count;
2563 last_wants = video_source_.sink_wants();
2564
2565 // Simulate the framerate we've been asked to adapt to.
2566 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2567 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2568 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2569 mock_stats.input_frame_rate = fps;
2570 stats_proxy_->SetMockStats(mock_stats);
2571
2572 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2573 sink_.WaitForEncodedFrame(t);
2574 t += frame_interval_ms;
2575
mflodmancc3d4422017-08-03 08:27:51 -07002576 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002577 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002578 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002579 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2580 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002581 } while (video_source_.sink_wants().max_pixel_count <
2582 last_wants.max_pixel_count ||
2583 video_source_.sink_wants().max_framerate_fps <
2584 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002585
Jonathan Yubc771b72017-12-08 17:04:29 -08002586 // Verify that we've adapted all the way down.
2587 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002588 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002589 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2590 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002591 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002592 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2593 *video_source_.last_sent_height());
2594 EXPECT_EQ(kMinBalancedFramerateFps,
2595 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002596
Jonathan Yubc771b72017-12-08 17:04:29 -08002597 // Adapt back up the same number of times we adapted down.
2598 for (int i = 0; i < loop_count - 1; ++i) {
2599 last_wants = video_source_.sink_wants();
2600
2601 // Simulate the framerate we've been asked to adapt to.
2602 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2603 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2604 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2605 mock_stats.input_frame_rate = fps;
2606 stats_proxy_->SetMockStats(mock_stats);
2607
2608 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2609 sink_.WaitForEncodedFrame(t);
2610 t += frame_interval_ms;
2611
Henrik Boström91aa7322020-04-28 12:24:33 +02002612 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002613 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002614 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002615 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2616 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002617 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2618 last_wants.max_pixel_count ||
2619 video_source_.sink_wants().max_framerate_fps >
2620 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002621 }
2622
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002623 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002624 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002625 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002626 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2627 EXPECT_EQ((loop_count - 1) * 2,
2628 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002629
mflodmancc3d4422017-08-03 08:27:51 -07002630 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002631}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002632
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002633TEST_F(VideoStreamEncoderTest,
2634 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02002635 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2636 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002637 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002638
2639 const int kFrameWidth = 1280;
2640 const int kFrameHeight = 720;
2641
2642 int64_t ntp_time = kFrameIntervalMs;
2643
2644 // Force an input frame rate to be available, or the adaptation call won't
2645 // know what framerate to adapt form.
2646 const int kInputFps = 30;
2647 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2648 stats.input_frame_rate = kInputFps;
2649 stats_proxy_->SetMockStats(stats);
2650
2651 video_source_.set_adaptation_enabled(true);
2652 video_stream_encoder_->SetSource(
2653 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002654 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002655 video_source_.IncomingCapturedFrame(
2656 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2657 sink_.WaitForEncodedFrame(ntp_time);
2658 ntp_time += kFrameIntervalMs;
2659
2660 // Trigger CPU overuse.
2661 video_stream_encoder_->TriggerCpuOveruse();
2662 video_source_.IncomingCapturedFrame(
2663 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2664 sink_.WaitForEncodedFrame(ntp_time);
2665 ntp_time += kFrameIntervalMs;
2666
2667 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2668 EXPECT_EQ(std::numeric_limits<int>::max(),
2669 video_source_.sink_wants().max_pixel_count);
2670 // Some framerate constraint should be set.
2671 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2672 EXPECT_LT(restricted_fps, kInputFps);
2673 video_source_.IncomingCapturedFrame(
2674 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2675 sink_.WaitForEncodedFrame(ntp_time);
2676 ntp_time += 100;
2677
Henrik Boström2671dac2020-05-19 16:29:09 +02002678 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002679 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2680 // Give the encoder queue time to process the change in degradation preference
2681 // by waiting for an encoded frame.
2682 video_source_.IncomingCapturedFrame(
2683 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2684 sink_.WaitForEncodedFrame(ntp_time);
2685 ntp_time += kFrameIntervalMs;
2686
2687 video_stream_encoder_->TriggerQualityLow();
2688 video_source_.IncomingCapturedFrame(
2689 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2690 sink_.WaitForEncodedFrame(ntp_time);
2691 ntp_time += kFrameIntervalMs;
2692
2693 // Some resolution constraint should be set.
2694 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2695 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2696 kFrameWidth * kFrameHeight);
2697 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2698
2699 int pixel_count = video_source_.sink_wants().max_pixel_count;
2700 // Triggering a CPU underuse should not change the sink wants since it has
2701 // not been overused for resolution since we changed degradation preference.
2702 video_stream_encoder_->TriggerCpuUnderuse();
2703 video_source_.IncomingCapturedFrame(
2704 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2705 sink_.WaitForEncodedFrame(ntp_time);
2706 ntp_time += kFrameIntervalMs;
2707 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2708 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2709
Evan Shrubsole64469032020-06-11 10:45:29 +02002710 // Change the degradation preference back. CPU underuse should not adapt since
2711 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002712 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002713 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2714 video_source_.IncomingCapturedFrame(
2715 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2716 sink_.WaitForEncodedFrame(ntp_time);
2717 ntp_time += 100;
2718 // Resolution adaptations is gone after changing degradation preference.
2719 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2720 EXPECT_EQ(std::numeric_limits<int>::max(),
2721 video_source_.sink_wants().max_pixel_count);
2722 // The fps adaptation from above is now back.
2723 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2724
2725 // Trigger CPU underuse.
2726 video_stream_encoder_->TriggerCpuUnderuse();
2727 video_source_.IncomingCapturedFrame(
2728 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2729 sink_.WaitForEncodedFrame(ntp_time);
2730 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002731 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2732
2733 // Trigger QP underuse, fps should return to normal.
2734 video_stream_encoder_->TriggerQualityHigh();
2735 video_source_.IncomingCapturedFrame(
2736 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2737 sink_.WaitForEncodedFrame(ntp_time);
2738 ntp_time += kFrameIntervalMs;
2739 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002740
2741 video_stream_encoder_->Stop();
2742}
2743
mflodmancc3d4422017-08-03 08:27:51 -07002744TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002745 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002746 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002747 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002748
sprangc5d62e22017-04-02 23:53:04 -07002749 const int kFrameWidth = 1280;
2750 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002751
Åsa Persson8c1bf952018-09-13 10:42:19 +02002752 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002753
kthelgason5e13d412016-12-01 03:59:51 -08002754 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002755 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002756 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002757 frame_timestamp += kFrameIntervalMs;
2758
perkj803d97f2016-11-01 11:45:46 -07002759 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002760 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002761 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002762 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002763 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002764 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002765
asapersson0944a802017-04-07 00:57:58 -07002766 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002767 // wanted resolution.
2768 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2769 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2770 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002771 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002772
2773 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002774 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002775 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002776 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002777 // Give the encoder queue time to process the change in degradation preference
2778 // by waiting for an encoded frame.
2779 new_video_source.IncomingCapturedFrame(
2780 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2781 sink_.WaitForEncodedFrame(frame_timestamp);
2782 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002783 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002784 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002785
sprangc5d62e22017-04-02 23:53:04 -07002786 // Force an input frame rate to be available, or the adaptation call won't
2787 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002788 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002789 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002790 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002791 stats_proxy_->SetMockStats(stats);
2792
mflodmancc3d4422017-08-03 08:27:51 -07002793 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002794 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002795 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002796 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002797 frame_timestamp += kFrameIntervalMs;
2798
2799 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002800 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002801 EXPECT_EQ(std::numeric_limits<int>::max(),
2802 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002803 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002804
asapersson02465b82017-04-10 01:12:52 -07002805 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002806 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2807 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002808 // Give the encoder queue time to process the change in degradation preference
2809 // by waiting for an encoded frame.
2810 new_video_source.IncomingCapturedFrame(
2811 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2812 sink_.WaitForEncodedFrame(frame_timestamp);
2813 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002814 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002815
mflodmancc3d4422017-08-03 08:27:51 -07002816 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002817 new_video_source.IncomingCapturedFrame(
2818 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002819 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002820 frame_timestamp += kFrameIntervalMs;
2821
2822 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002823 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002824
2825 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002826 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002827 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002828 // Give the encoder queue time to process the change in degradation preference
2829 // by waiting for an encoded frame.
2830 new_video_source.IncomingCapturedFrame(
2831 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2832 sink_.WaitForEncodedFrame(frame_timestamp);
2833 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002834 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2835 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002836 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002837 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002838
2839 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002840 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002841 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002842 // Give the encoder queue time to process the change in degradation preference
2843 // by waiting for an encoded frame.
2844 new_video_source.IncomingCapturedFrame(
2845 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2846 sink_.WaitForEncodedFrame(frame_timestamp);
2847 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002848 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2849 EXPECT_EQ(std::numeric_limits<int>::max(),
2850 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002851 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002852
mflodmancc3d4422017-08-03 08:27:51 -07002853 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002854}
2855
mflodmancc3d4422017-08-03 08:27:51 -07002856TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002857 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002858 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002859
asaperssonfab67072017-04-04 05:51:49 -07002860 const int kWidth = 1280;
2861 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002862 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002863 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002864 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2865 EXPECT_FALSE(stats.bw_limited_resolution);
2866 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2867
2868 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002869 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002870 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002871 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002872
2873 stats = stats_proxy_->GetStats();
2874 EXPECT_TRUE(stats.bw_limited_resolution);
2875 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2876
2877 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002878 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002879 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002880 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002881
2882 stats = stats_proxy_->GetStats();
2883 EXPECT_FALSE(stats.bw_limited_resolution);
2884 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2885 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2886
mflodmancc3d4422017-08-03 08:27:51 -07002887 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002888}
2889
mflodmancc3d4422017-08-03 08:27:51 -07002890TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002891 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002892 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002893
2894 const int kWidth = 1280;
2895 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002896 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002897 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002898 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2899 EXPECT_FALSE(stats.cpu_limited_resolution);
2900 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2901
2902 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002903 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002904 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002905 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002906
2907 stats = stats_proxy_->GetStats();
2908 EXPECT_TRUE(stats.cpu_limited_resolution);
2909 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2910
2911 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002912 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002913 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002914 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002915
2916 stats = stats_proxy_->GetStats();
2917 EXPECT_FALSE(stats.cpu_limited_resolution);
2918 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002919 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002920
mflodmancc3d4422017-08-03 08:27:51 -07002921 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002922}
2923
mflodmancc3d4422017-08-03 08:27:51 -07002924TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002925 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002926 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002927
asaperssonfab67072017-04-04 05:51:49 -07002928 const int kWidth = 1280;
2929 const int kHeight = 720;
2930 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002931 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002932 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002933 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002934 EXPECT_FALSE(stats.cpu_limited_resolution);
2935 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2936
asaperssonfab67072017-04-04 05:51:49 -07002937 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002938 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002939 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002940 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002941 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002942 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002943 EXPECT_TRUE(stats.cpu_limited_resolution);
2944 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2945
2946 // Set new source with adaptation still enabled.
2947 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002948 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002949 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002950
asaperssonfab67072017-04-04 05:51:49 -07002951 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002952 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002953 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002954 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002955 EXPECT_TRUE(stats.cpu_limited_resolution);
2956 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2957
2958 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002959 video_stream_encoder_->SetSource(&new_video_source,
2960 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002961
asaperssonfab67072017-04-04 05:51:49 -07002962 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002963 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002964 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002965 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002966 EXPECT_FALSE(stats.cpu_limited_resolution);
2967 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2968
2969 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002970 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002971 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002972
asaperssonfab67072017-04-04 05:51:49 -07002973 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002974 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002975 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002976 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002977 EXPECT_TRUE(stats.cpu_limited_resolution);
2978 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2979
asaperssonfab67072017-04-04 05:51:49 -07002980 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002981 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002982 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002983 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002984 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002985 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002986 EXPECT_FALSE(stats.cpu_limited_resolution);
2987 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002988 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002989
mflodmancc3d4422017-08-03 08:27:51 -07002990 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002991}
2992
mflodmancc3d4422017-08-03 08:27:51 -07002993TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002994 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002995 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002996
asaperssonfab67072017-04-04 05:51:49 -07002997 const int kWidth = 1280;
2998 const int kHeight = 720;
2999 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003000 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003001 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003002 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003003 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003004 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003005
3006 // Set new source with adaptation still enabled.
3007 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003008 video_stream_encoder_->SetSource(&new_video_source,
3009 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08003010
asaperssonfab67072017-04-04 05:51:49 -07003011 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003012 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08003013 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003014 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003015 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003016 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003017
asaperssonfab67072017-04-04 05:51:49 -07003018 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003019 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003020 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003021 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003022 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003023 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003024 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003025 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003026
asaperssonfab67072017-04-04 05:51:49 -07003027 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003028 video_stream_encoder_->SetSource(&new_video_source,
3029 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08003030
asaperssonfab67072017-04-04 05:51:49 -07003031 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003032 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003033 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003034 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003035 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003036 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003037
asapersson02465b82017-04-10 01:12:52 -07003038 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07003039 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003040 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003041
asaperssonfab67072017-04-04 05:51:49 -07003042 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003043 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08003044 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003045 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003046 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003047 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
3048 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003049
mflodmancc3d4422017-08-03 08:27:51 -07003050 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003051}
3052
mflodmancc3d4422017-08-03 08:27:51 -07003053TEST_F(VideoStreamEncoderTest,
3054 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02003055 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003056 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07003057
3058 const int kWidth = 1280;
3059 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02003060 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07003061 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003062 video_source_.IncomingCapturedFrame(
3063 CreateFrame(timestamp_ms, kWidth, kHeight));
3064 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003065 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3066 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3067 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3068
3069 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003070 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003071 timestamp_ms += kFrameIntervalMs;
3072 video_source_.IncomingCapturedFrame(
3073 CreateFrame(timestamp_ms, kWidth, kHeight));
3074 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003075 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3076 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3077 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3078
3079 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07003080 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003081 timestamp_ms += kFrameIntervalMs;
3082 video_source_.IncomingCapturedFrame(
3083 CreateFrame(timestamp_ms, kWidth, kHeight));
3084 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003085 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3086 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3087 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3088
Niels Möller4db138e2018-04-19 09:04:13 +02003089 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07003090 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003091
3092 VideoEncoderConfig video_encoder_config;
3093 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3094 // Make format different, to force recreation of encoder.
3095 video_encoder_config.video_format.parameters["foo"] = "foo";
3096 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003097 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003098 timestamp_ms += kFrameIntervalMs;
3099 video_source_.IncomingCapturedFrame(
3100 CreateFrame(timestamp_ms, kWidth, kHeight));
3101 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003102 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3103 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3104 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3105
mflodmancc3d4422017-08-03 08:27:51 -07003106 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003107}
3108
mflodmancc3d4422017-08-03 08:27:51 -07003109TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003110 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003111 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003112 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003113
3114 const int kWidth = 1280;
3115 const int kHeight = 720;
3116 int sequence = 1;
3117
3118 // Enable BALANCED preference, no initial limitation.
3119 test::FrameForwarder source;
3120 video_stream_encoder_->SetSource(&source,
3121 webrtc::DegradationPreference::BALANCED);
3122 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3123 WaitForEncodedFrame(sequence++);
3124 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3125 EXPECT_FALSE(stats.cpu_limited_resolution);
3126 EXPECT_FALSE(stats.cpu_limited_framerate);
3127 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3128
3129 // Trigger CPU overuse, should now adapt down.
3130 video_stream_encoder_->TriggerCpuOveruse();
3131 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3132 WaitForEncodedFrame(sequence++);
3133 stats = stats_proxy_->GetStats();
3134 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3135
3136 // Set new degradation preference should clear restrictions since we changed
3137 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003138 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003139 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3140 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3141 WaitForEncodedFrame(sequence++);
3142 stats = stats_proxy_->GetStats();
3143 EXPECT_FALSE(stats.cpu_limited_resolution);
3144 EXPECT_FALSE(stats.cpu_limited_framerate);
3145 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3146
3147 // Force an input frame rate to be available, or the adaptation call won't
3148 // know what framerate to adapt from.
3149 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3150 mock_stats.input_frame_rate = 30;
3151 stats_proxy_->SetMockStats(mock_stats);
3152 video_stream_encoder_->TriggerCpuOveruse();
3153 stats_proxy_->ResetMockStats();
3154 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3155 WaitForEncodedFrame(sequence++);
3156
3157 // We have now adapted once.
3158 stats = stats_proxy_->GetStats();
3159 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3160
3161 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003162 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3163 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003164 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3165 WaitForEncodedFrame(sequence++);
3166 stats = stats_proxy_->GetStats();
3167 EXPECT_FALSE(stats.cpu_limited_resolution);
3168 EXPECT_FALSE(stats.cpu_limited_framerate);
3169 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3170
3171 video_stream_encoder_->Stop();
3172}
3173
3174TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003175 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003176 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003177 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003178
asapersson0944a802017-04-07 00:57:58 -07003179 const int kWidth = 1280;
3180 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003181 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003182
asaperssonfab67072017-04-04 05:51:49 -07003183 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003184 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003185 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003186 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003187 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003188 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3189
asapersson02465b82017-04-10 01:12:52 -07003190 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003191 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003192 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003193 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003194 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003195 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003196 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003197 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3198
3199 // Set new source with adaptation still enabled.
3200 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003201 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003202 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003203
3204 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003205 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003206 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003207 stats = stats_proxy_->GetStats();
3208 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003209 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003210 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3211
sprangc5d62e22017-04-02 23:53:04 -07003212 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003213 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003214 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003215 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003216 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003217 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003218 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003219 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003220 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003221 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003222 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3223
sprangc5d62e22017-04-02 23:53:04 -07003224 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003225 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003226 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3227 mock_stats.input_frame_rate = 30;
3228 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003229 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003230 stats_proxy_->ResetMockStats();
3231
3232 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003233 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003234 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003235
3236 // Framerate now adapted.
3237 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003238 EXPECT_FALSE(stats.cpu_limited_resolution);
3239 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003240 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3241
3242 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003243 video_stream_encoder_->SetSource(&new_video_source,
3244 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003245 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003246 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003247 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003248
3249 stats = stats_proxy_->GetStats();
3250 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003251 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003252 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3253
3254 // Try to trigger overuse. Should not succeed.
3255 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003256 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003257 stats_proxy_->ResetMockStats();
3258
3259 stats = stats_proxy_->GetStats();
3260 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003261 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003262 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3263
3264 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003265 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003266 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003267 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003268 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003269 stats = stats_proxy_->GetStats();
3270 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003271 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003272 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003273
3274 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003275 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003276 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003277 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003278 stats = stats_proxy_->GetStats();
3279 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003280 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003281 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3282
3283 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003284 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003285 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003286 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003287 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003288 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003289 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003290 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003291 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003292 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003293 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3294
3295 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003296 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003297 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003298 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003299 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003300 stats = stats_proxy_->GetStats();
3301 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003302 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003303 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003304 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003305
mflodmancc3d4422017-08-03 08:27:51 -07003306 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003307}
3308
mflodmancc3d4422017-08-03 08:27:51 -07003309TEST_F(VideoStreamEncoderTest,
3310 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003311 const int kWidth = 1280;
3312 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003313 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003314 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003315
asaperssonfab67072017-04-04 05:51:49 -07003316 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003317 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003318
asaperssonfab67072017-04-04 05:51:49 -07003319 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003320 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003321
asaperssonfab67072017-04-04 05:51:49 -07003322 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003323 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003324
asaperssonfab67072017-04-04 05:51:49 -07003325 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003326 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003327
kthelgason876222f2016-11-29 01:44:11 -08003328 // Expect a scale down.
3329 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003330 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003331
asapersson02465b82017-04-10 01:12:52 -07003332 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003333 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003334 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003335 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003336
asaperssonfab67072017-04-04 05:51:49 -07003337 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003338 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003339 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003340 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003341
asaperssonfab67072017-04-04 05:51:49 -07003342 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003343 EXPECT_EQ(std::numeric_limits<int>::max(),
3344 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003345
asaperssonfab67072017-04-04 05:51:49 -07003346 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003347 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003348 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003349 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003350
asapersson02465b82017-04-10 01:12:52 -07003351 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003352 EXPECT_EQ(std::numeric_limits<int>::max(),
3353 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003354
mflodmancc3d4422017-08-03 08:27:51 -07003355 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003356}
3357
mflodmancc3d4422017-08-03 08:27:51 -07003358TEST_F(VideoStreamEncoderTest,
3359 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003360 const int kWidth = 1280;
3361 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003362 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003363 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003364
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003365 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003366 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003367 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003368 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003369
3370 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003371 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003372 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003373 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3374 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3375
3376 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003377 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003378 EXPECT_THAT(source.sink_wants(),
3379 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003380 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3381 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3382 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3383
3384 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003385 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003386 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3387 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3388 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3389
mflodmancc3d4422017-08-03 08:27:51 -07003390 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003391}
3392
mflodmancc3d4422017-08-03 08:27:51 -07003393TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003394 const int kWidth = 1280;
3395 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003396 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003397 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003398
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003399 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003400 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003401 video_stream_encoder_->SetSource(&source,
3402 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003403 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3404 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003405 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003406
3407 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003408 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003409 EXPECT_THAT(source.sink_wants(),
3410 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003411 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3412 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3413 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3414
3415 // Trigger adapt down for same input resolution, expect no change.
3416 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3417 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003418 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003419 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3420 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3421 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3422
3423 // Trigger adapt down for larger input resolution, expect no change.
3424 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3425 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003426 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003427 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3428 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3429 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3430
mflodmancc3d4422017-08-03 08:27:51 -07003431 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003432}
3433
mflodmancc3d4422017-08-03 08:27:51 -07003434TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003435 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3436 const int kWidth = 640;
3437 const int kHeight = 360;
3438 const int64_t kFrameIntervalMs = 150;
3439 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003440 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003441
3442 // Enable BALANCED preference, no initial limitation.
3443 AdaptingFrameForwarder source(&time_controller_);
3444 source.set_adaptation_enabled(true);
3445 video_stream_encoder_->SetSource(&source,
3446 webrtc::DegradationPreference::BALANCED);
3447
3448 int64_t timestamp_ms = kFrameIntervalMs;
3449 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3450 sink_.WaitForEncodedFrame(kWidth, kHeight);
3451 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3452 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3453 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3454 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3455
3456 // Trigger adapt down, expect reduced fps (640x360@15fps).
3457 video_stream_encoder_->TriggerQualityLow();
3458 timestamp_ms += kFrameIntervalMs;
3459 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3460 sink_.WaitForEncodedFrame(timestamp_ms);
3461 EXPECT_THAT(source.sink_wants(),
3462 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3463 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3464 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3465 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3466
3467 // Source requests 270p, expect reduced resolution (480x270@15fps).
3468 source.OnOutputFormatRequest(480, 270);
3469 timestamp_ms += kFrameIntervalMs;
3470 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3471 WaitForEncodedFrame(480, 270);
3472 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3473
3474 // Trigger adapt down, expect reduced fps (480x270@10fps).
3475 video_stream_encoder_->TriggerQualityLow();
3476 timestamp_ms += kFrameIntervalMs;
3477 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3478 sink_.WaitForEncodedFrame(timestamp_ms);
3479 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3480 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3481 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3482 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3483
3484 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3485 source.OnOutputFormatRequest(320, 180);
3486 timestamp_ms += kFrameIntervalMs;
3487 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3488 WaitForEncodedFrame(320, 180);
3489 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3490
3491 // Trigger adapt down, expect reduced fps (320x180@7fps).
3492 video_stream_encoder_->TriggerQualityLow();
3493 timestamp_ms += kFrameIntervalMs;
3494 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3495 sink_.WaitForEncodedFrame(timestamp_ms);
3496 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3497 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3498 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3499 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3500
3501 // Source requests VGA, expect increased resolution (640x360@7fps).
3502 source.OnOutputFormatRequest(640, 360);
3503 timestamp_ms += kFrameIntervalMs;
3504 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3505 WaitForEncodedFrame(timestamp_ms);
3506 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3507
3508 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3509 video_stream_encoder_->TriggerQualityHigh();
3510 timestamp_ms += kFrameIntervalMs;
3511 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3512 WaitForEncodedFrame(timestamp_ms);
3513 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3514 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3515 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3516 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3517
3518 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3519 video_stream_encoder_->TriggerQualityHigh();
3520 timestamp_ms += kFrameIntervalMs;
3521 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3522 WaitForEncodedFrame(timestamp_ms);
3523 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3524 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3525 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3526 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3527
3528 // Trigger adapt up, expect increased fps (640x360@maxfps).
3529 video_stream_encoder_->TriggerQualityHigh();
3530 timestamp_ms += kFrameIntervalMs;
3531 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3532 WaitForEncodedFrame(timestamp_ms);
3533 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3534 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3535 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3536 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3537
3538 video_stream_encoder_->Stop();
3539}
3540
3541TEST_F(VideoStreamEncoderTest,
3542 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3543 const int kWidth = 1280;
3544 const int kHeight = 720;
3545 const int64_t kFrameIntervalMs = 150;
3546 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003547 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003548
3549 // Enable BALANCED preference, no initial limitation.
3550 AdaptingFrameForwarder source(&time_controller_);
3551 source.set_adaptation_enabled(true);
3552 video_stream_encoder_->SetSource(&source,
3553 webrtc::DegradationPreference::BALANCED);
3554
3555 int64_t timestamp_ms = kFrameIntervalMs;
3556 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3557 sink_.WaitForEncodedFrame(kWidth, kHeight);
3558 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3559 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3560 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3561 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3562
3563 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3564 video_stream_encoder_->TriggerQualityLow();
3565 timestamp_ms += kFrameIntervalMs;
3566 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3567 sink_.WaitForEncodedFrame(timestamp_ms);
3568 EXPECT_THAT(source.sink_wants(),
3569 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3570 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3571 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3572 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3573
3574 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3575 video_stream_encoder_->TriggerQualityLow();
3576 timestamp_ms += kFrameIntervalMs;
3577 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3578 sink_.WaitForEncodedFrame(timestamp_ms);
3579 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3580 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3581 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3582 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3583
3584 // Trigger adapt down, expect reduced fps (640x360@15fps).
3585 video_stream_encoder_->TriggerQualityLow();
3586 timestamp_ms += kFrameIntervalMs;
3587 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3588 WaitForEncodedFrame(timestamp_ms);
3589 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3590 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3591 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3592 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3593
3594 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3595 source.OnOutputFormatRequest(320, 180);
3596 timestamp_ms += kFrameIntervalMs;
3597 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3598 WaitForEncodedFrame(320, 180);
3599 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3600 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3601
3602 // Trigger adapt down, expect reduced fps (320x180@7fps).
3603 video_stream_encoder_->TriggerCpuOveruse();
3604 timestamp_ms += kFrameIntervalMs;
3605 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3606 WaitForEncodedFrame(timestamp_ms);
3607 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3608 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3609 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3610 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3611 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3612 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3613 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3614
3615 // Source requests HD, expect increased resolution (640x360@7fps).
3616 source.OnOutputFormatRequest(1280, 720);
3617 timestamp_ms += kFrameIntervalMs;
3618 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3619 WaitForEncodedFrame(timestamp_ms);
3620 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3621 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3622
3623 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3624 video_stream_encoder_->TriggerCpuUnderuse();
3625 timestamp_ms += kFrameIntervalMs;
3626 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3627 WaitForEncodedFrame(timestamp_ms);
3628 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3629 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3630 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3631 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3632 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3633 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3634 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3635
3636 // Trigger adapt up, expect increased fps (640x360@maxfps).
3637 video_stream_encoder_->TriggerQualityHigh();
3638 video_stream_encoder_->TriggerCpuUnderuse();
3639 timestamp_ms += kFrameIntervalMs;
3640 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3641 WaitForEncodedFrame(timestamp_ms);
3642 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3643 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3644 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3645 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3646 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3647 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3648 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3649
3650 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3651 video_stream_encoder_->TriggerQualityHigh();
3652 video_stream_encoder_->TriggerCpuUnderuse();
3653 timestamp_ms += kFrameIntervalMs;
3654 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3655 WaitForEncodedFrame(timestamp_ms);
3656 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3657 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3658 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3659 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3660 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3661 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3662 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3663
3664 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3665 video_stream_encoder_->TriggerQualityHigh();
3666 video_stream_encoder_->TriggerCpuUnderuse();
3667 timestamp_ms += kFrameIntervalMs;
3668 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3669 WaitForEncodedFrame(timestamp_ms);
3670 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3671 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3672 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3673 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3674 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3675 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3676 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3677
3678 video_stream_encoder_->Stop();
3679}
3680
3681TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003682 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003683 const int kWidth = 1280;
3684 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003685 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003686 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003687
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003688 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003689 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003690 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003691 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003692
3693 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003694 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003695 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003696 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3697 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3698
3699 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003700 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003701 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003702 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3703 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3704
mflodmancc3d4422017-08-03 08:27:51 -07003705 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003706}
3707
mflodmancc3d4422017-08-03 08:27:51 -07003708TEST_F(VideoStreamEncoderTest,
3709 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003710 const int kWidth = 1280;
3711 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003712 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003713 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003714
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003715 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003716 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003717 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003718 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003719
3720 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003721 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003722 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003723 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003724 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3725
3726 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003727 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003728 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003729 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003730 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3731
mflodmancc3d4422017-08-03 08:27:51 -07003732 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003733}
3734
mflodmancc3d4422017-08-03 08:27:51 -07003735TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003736 const int kWidth = 1280;
3737 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003738 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003739 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003740
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003741 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003742 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003743 video_stream_encoder_->SetSource(&source,
3744 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003745
3746 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3747 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003748 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003749 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3750 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3751 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3752
3753 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003754 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003755 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003756 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3757 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3758 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3759
mflodmancc3d4422017-08-03 08:27:51 -07003760 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003761}
3762
mflodmancc3d4422017-08-03 08:27:51 -07003763TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003764 const int kWidth = 1280;
3765 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003766 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003767 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003768
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003769 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003770 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003771 video_stream_encoder_->SetSource(&source,
3772 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003773
3774 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3775 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003776 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003777 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3778 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3779 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3780
3781 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003782 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003783 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003784 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3785 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3786 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3787
mflodmancc3d4422017-08-03 08:27:51 -07003788 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003789}
3790
mflodmancc3d4422017-08-03 08:27:51 -07003791TEST_F(VideoStreamEncoderTest,
3792 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003793 const int kWidth = 1280;
3794 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003795 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003796 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003797
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003798 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003799 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003800 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003801 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003802 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003803
3804 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003805 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003806 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003807 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3808 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3809
3810 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003811 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003812 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003813 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003814 EXPECT_THAT(source.sink_wants(),
3815 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003816 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3817 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3818
3819 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003820 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003821 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003822 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3823 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3824 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3825
mflodmancc3d4422017-08-03 08:27:51 -07003826 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003827}
3828
mflodmancc3d4422017-08-03 08:27:51 -07003829TEST_F(VideoStreamEncoderTest,
3830 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003831 const int kWidth = 1280;
3832 const int kHeight = 720;
3833 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003834 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003835 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003836
3837 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3838 stats.input_frame_rate = kInputFps;
3839 stats_proxy_->SetMockStats(stats);
3840
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003841 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003842 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3843 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003844 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003845
3846 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003847 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003848 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3849 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003850 EXPECT_THAT(video_source_.sink_wants(),
3851 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003852
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003853 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003854 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003855 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003856 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003857 // Give the encoder queue time to process the change in degradation preference
3858 // by waiting for an encoded frame.
3859 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3860 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003861 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003862
3863 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003864 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003865 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3866 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003867 EXPECT_THAT(new_video_source.sink_wants(),
3868 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003869
3870 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003871 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003872 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003873
mflodmancc3d4422017-08-03 08:27:51 -07003874 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003875}
3876
mflodmancc3d4422017-08-03 08:27:51 -07003877TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003878 const int kWidth = 1280;
3879 const int kHeight = 720;
3880 const size_t kNumFrames = 10;
3881
Henrik Boström381d1092020-05-12 18:49:07 +02003882 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003883 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003884
asaperssond0de2952017-04-21 01:47:31 -07003885 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003886 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003887 video_source_.set_adaptation_enabled(true);
3888
3889 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3890 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3891
3892 int downscales = 0;
3893 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003894 video_source_.IncomingCapturedFrame(
3895 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3896 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003897
asaperssonfab67072017-04-04 05:51:49 -07003898 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003899 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003900 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003901 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003902
3903 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3904 ++downscales;
3905
3906 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3907 EXPECT_EQ(downscales,
3908 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3909 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003910 }
mflodmancc3d4422017-08-03 08:27:51 -07003911 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003912}
3913
mflodmancc3d4422017-08-03 08:27:51 -07003914TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003915 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3916 const int kWidth = 1280;
3917 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003918 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003919 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003920
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003921 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003922 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003923 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003924 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003925 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003926
Åsa Persson8c1bf952018-09-13 10:42:19 +02003927 int64_t timestamp_ms = kFrameIntervalMs;
3928 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003929 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003930 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003931 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3932 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3933
3934 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003935 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003936 timestamp_ms += kFrameIntervalMs;
3937 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3938 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003939 EXPECT_THAT(source.sink_wants(),
3940 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003941 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3942 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3943
3944 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003945 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003946 timestamp_ms += kFrameIntervalMs;
3947 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003948 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003949 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003950 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3951 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3952
3953 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003954 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003955 timestamp_ms += kFrameIntervalMs;
3956 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3957 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003958 EXPECT_THAT(source.sink_wants(),
3959 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003960 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3961 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3962
3963 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003964 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003965 timestamp_ms += kFrameIntervalMs;
3966 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003967 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003968 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003969 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3970 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3971
mflodmancc3d4422017-08-03 08:27:51 -07003972 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003973}
3974
mflodmancc3d4422017-08-03 08:27:51 -07003975TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003976 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3977 const int kWidth = 1280;
3978 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003979 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003980 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003981
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003982 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003983 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003984 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003985 video_stream_encoder_->SetSource(&source,
3986 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003987
Åsa Persson8c1bf952018-09-13 10:42:19 +02003988 int64_t timestamp_ms = kFrameIntervalMs;
3989 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003990 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003991 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003992 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3993 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3994
3995 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003996 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003997 timestamp_ms += kFrameIntervalMs;
3998 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3999 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004000 EXPECT_THAT(source.sink_wants(),
4001 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004002 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4003 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4004
4005 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07004006 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004007 timestamp_ms += kFrameIntervalMs;
4008 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07004009 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004010 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004011 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4012 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4013
4014 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07004015 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004016 timestamp_ms += kFrameIntervalMs;
4017 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4018 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004019 EXPECT_THAT(source.sink_wants(),
4020 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004021 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4022 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4023
4024 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07004025 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004026 timestamp_ms += kFrameIntervalMs;
4027 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07004028 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004029 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004030 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4031 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4032
mflodmancc3d4422017-08-03 08:27:51 -07004033 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004034}
4035
Sergey Silkin41c650b2019-10-14 13:12:19 +02004036TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
4037 fake_encoder_.SetResolutionBitrateLimits(
4038 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4039
Henrik Boström381d1092020-05-12 18:49:07 +02004040 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004041 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4042 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4043 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4044 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004045
4046 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004047 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004048 source.set_adaptation_enabled(true);
4049 video_stream_encoder_->SetSource(
4050 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4051
4052 // Insert 720p frame.
4053 int64_t timestamp_ms = kFrameIntervalMs;
4054 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4055 WaitForEncodedFrame(1280, 720);
4056
4057 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02004058 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004059 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4060 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4061 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4062 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004063 video_stream_encoder_->TriggerQualityLow();
4064
4065 // Insert 720p frame. It should be downscaled and encoded.
4066 timestamp_ms += kFrameIntervalMs;
4067 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4068 WaitForEncodedFrame(960, 540);
4069
4070 // Trigger adapt up. Higher resolution should not be requested duo to lack
4071 // of bitrate.
4072 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004073 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02004074
4075 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02004076 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004077 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4078 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4079 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4080 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004081
4082 // Trigger adapt up. Higher resolution should be requested.
4083 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004084 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02004085
4086 video_stream_encoder_->Stop();
4087}
4088
4089TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
4090 fake_encoder_.SetResolutionBitrateLimits(
4091 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4092
4093 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02004094 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004095 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4096 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4097 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4098 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004099
4100 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004101 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004102 source.set_adaptation_enabled(true);
4103 video_stream_encoder_->SetSource(
4104 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4105
4106 // Insert 720p frame. It should be dropped and lower resolution should be
4107 // requested.
4108 int64_t timestamp_ms = kFrameIntervalMs;
4109 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4110 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004111 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004112
4113 // Insert 720p frame. It should be downscaled and encoded.
4114 timestamp_ms += kFrameIntervalMs;
4115 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4116 WaitForEncodedFrame(960, 540);
4117
4118 video_stream_encoder_->Stop();
4119}
4120
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004121class BalancedDegradationTest : public VideoStreamEncoderTest {
4122 protected:
4123 void SetupTest() {
4124 // Reset encoder for field trials to take effect.
4125 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02004126 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004127
4128 // Enable BALANCED preference.
4129 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02004130 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4131 }
4132
Asa Persson606d3cb2021-10-04 10:07:11 +02004133 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02004134 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004135 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004136 }
4137
Åsa Persson45b176f2019-09-30 11:19:05 +02004138 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004139 timestamp_ms_ += kFrameIntervalMs;
4140 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02004141 }
4142
4143 void InsertFrameAndWaitForEncoded() {
4144 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004145 sink_.WaitForEncodedFrame(timestamp_ms_);
4146 }
4147
4148 const int kWidth = 640; // pixels:640x360=230400
4149 const int kHeight = 360;
4150 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4151 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004152 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004153};
4154
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004155TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004156 test::ScopedKeyValueConfig field_trials(
4157 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004158 "WebRTC-Video-BalancedDegradationSettings/"
4159 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4160 SetupTest();
4161
4162 // Force input frame rate.
4163 const int kInputFps = 24;
4164 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4165 stats.input_frame_rate = kInputFps;
4166 stats_proxy_->SetMockStats(stats);
4167
Åsa Persson45b176f2019-09-30 11:19:05 +02004168 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004169 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004170
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004171 // Trigger adapt down, expect scaled down framerate and resolution,
4172 // since Fps diff (input-requested:0) < threshold.
4173 video_stream_encoder_->TriggerQualityLow();
4174 EXPECT_THAT(source_.sink_wants(),
4175 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004176
4177 video_stream_encoder_->Stop();
4178}
4179
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004180TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004181 test::ScopedKeyValueConfig field_trials(
4182 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004183 "WebRTC-Video-BalancedDegradationSettings/"
4184 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4185 SetupTest();
4186
4187 // Force input frame rate.
4188 const int kInputFps = 25;
4189 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4190 stats.input_frame_rate = kInputFps;
4191 stats_proxy_->SetMockStats(stats);
4192
Åsa Persson45b176f2019-09-30 11:19:05 +02004193 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004194 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004195
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004196 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4197 // Fps diff (input-requested:1) == threshold.
4198 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004199 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004200
4201 video_stream_encoder_->Stop();
4202}
4203
4204TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004205 test::ScopedKeyValueConfig field_trials(
4206 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004207 "WebRTC-Video-BalancedDegradationSettings/"
4208 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4209 SetupTest();
4210
4211 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4212
Åsa Persson45b176f2019-09-30 11:19:05 +02004213 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004214 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004215
4216 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4217 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004218 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004219
4220 video_stream_encoder_->Stop();
4221}
4222
Åsa Perssonccfb3402019-09-25 15:13:04 +02004223TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004224 test::ScopedKeyValueConfig field_trials(
4225 field_trials_,
Åsa Persson1b247f12019-08-14 17:26:39 +02004226 "WebRTC-Video-BalancedDegradationSettings/"
4227 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004228 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004229
Asa Persson606d3cb2021-10-04 10:07:11 +02004230 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4231 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4232 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004233
Åsa Persson45b176f2019-09-30 11:19:05 +02004234 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004235 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004236 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4237
4238 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4239 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004240 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004241 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004242 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4243
4244 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4245 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004246 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004247 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004248 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4249
Åsa Persson30ab0152019-08-27 12:22:33 +02004250 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4251 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004252 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004253 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004254 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004255 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4256
4257 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004258 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004259 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004260 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004261
Åsa Persson30ab0152019-08-27 12:22:33 +02004262 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004263 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004264 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004265 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004266 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004267 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4268
4269 video_stream_encoder_->Stop();
4270}
4271
Åsa Perssonccfb3402019-09-25 15:13:04 +02004272TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004273 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004274 test::ScopedKeyValueConfig field_trials(
4275 field_trials_,
Åsa Persson45b176f2019-09-30 11:19:05 +02004276 "WebRTC-Video-BalancedDegradationSettings/"
4277 "pixels:57600|129600|230400,fps:7|24|24/");
4278 SetupTest();
Asa Persson606d3cb2021-10-04 10:07:11 +02004279 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 11:19:05 +02004280
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004281 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004282
4283 // Insert frame, expect scaled down:
4284 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4285 InsertFrame();
4286 EXPECT_FALSE(WaitForFrame(1000));
4287 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4288 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4289
4290 // Insert frame, expect scaled down:
4291 // resolution (320x180@24fps).
4292 InsertFrame();
4293 EXPECT_FALSE(WaitForFrame(1000));
4294 EXPECT_LT(source_.sink_wants().max_pixel_count,
4295 source_.last_wants().max_pixel_count);
4296 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4297
4298 // Frame should not be dropped (min pixels per frame reached).
4299 InsertFrameAndWaitForEncoded();
4300
4301 video_stream_encoder_->Stop();
4302}
4303
4304TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004305 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004306 test::ScopedKeyValueConfig field_trials(
4307 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004308 "WebRTC-Video-BalancedDegradationSettings/"
4309 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004310 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004311
Asa Persson606d3cb2021-10-04 10:07:11 +02004312 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4313 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4314 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004315
Åsa Persson45b176f2019-09-30 11:19:05 +02004316 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004317 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004318 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4319
4320 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4321 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004322 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004323 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004324 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4325
4326 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4327 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004328 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004329 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004330 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4331
4332 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4333 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004334 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004335 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004336 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4337
Åsa Persson30ab0152019-08-27 12:22:33 +02004338 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4339 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004340 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004341 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004342 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4343
4344 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4345 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004346 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004347 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4348
4349 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004350 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004351 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004352 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004353 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004354 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4355
4356 video_stream_encoder_->Stop();
4357}
4358
Åsa Perssonccfb3402019-09-25 15:13:04 +02004359TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004360 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004361 test::ScopedKeyValueConfig field_trials(
4362 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004363 "WebRTC-Video-BalancedDegradationSettings/"
4364 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004365 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004366
Asa Persson606d3cb2021-10-04 10:07:11 +02004367 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4368 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4369 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4370 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4371 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004372
Åsa Persson45b176f2019-09-30 11:19:05 +02004373 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004374 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004375 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4376
4377 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4378 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004379 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004380 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004381 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4382
4383 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4384 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004385 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004386 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004387 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4388
4389 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4390 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004391 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004392 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004393 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4394
4395 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4396 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004397 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004398 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4399
4400 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004401 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004402 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004403 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004404 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004405 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4406
4407 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004408 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004409 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004410 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004411 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4412
4413 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004414 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004415 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004416 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004417 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004418 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4419
Åsa Persson1b247f12019-08-14 17:26:39 +02004420 video_stream_encoder_->Stop();
4421}
4422
mflodmancc3d4422017-08-03 08:27:51 -07004423TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004424 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4425 const int kWidth = 1280;
4426 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004427 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004428 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004429
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004430 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004431 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004432 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004433 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004434 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004435
Åsa Persson8c1bf952018-09-13 10:42:19 +02004436 int64_t timestamp_ms = kFrameIntervalMs;
4437 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004438 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004439 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004440 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4441 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4442 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4443 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4444
4445 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004446 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004447 timestamp_ms += kFrameIntervalMs;
4448 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4449 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004450 EXPECT_THAT(source.sink_wants(),
4451 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004452 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4453 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4454 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4455 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4456
4457 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004458 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004459 timestamp_ms += kFrameIntervalMs;
4460 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4461 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004462 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004463 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4464 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4465 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4466 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4467
Jonathan Yubc771b72017-12-08 17:04:29 -08004468 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004469 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004470 timestamp_ms += kFrameIntervalMs;
4471 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4472 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004473 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004474 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4475 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004476 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004477 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4478
Jonathan Yubc771b72017-12-08 17:04:29 -08004479 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004480 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004481 timestamp_ms += kFrameIntervalMs;
4482 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4483 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004484 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004485 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004486 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4487 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4488 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4489 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4490
Jonathan Yubc771b72017-12-08 17:04:29 -08004491 // Trigger quality adapt down, expect no change (min resolution reached).
4492 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004493 timestamp_ms += kFrameIntervalMs;
4494 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4495 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004496 EXPECT_THAT(source.sink_wants(), FpsMax());
4497 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004498 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4499 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4500 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4501 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4502
Evan Shrubsole64469032020-06-11 10:45:29 +02004503 // Trigger quality adapt up, expect upscaled resolution (480x270).
4504 video_stream_encoder_->TriggerQualityHigh();
4505 timestamp_ms += kFrameIntervalMs;
4506 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4507 WaitForEncodedFrame(timestamp_ms);
4508 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4509 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4510 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4511 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4512 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4513
4514 // Trigger quality and cpu adapt up since both are most limited, expect
4515 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004516 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004517 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004518 timestamp_ms += kFrameIntervalMs;
4519 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4520 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004521 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004522 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4523 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4524 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004525 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004526
Evan Shrubsole64469032020-06-11 10:45:29 +02004527 // Trigger quality and cpu adapt up since both are most limited, expect
4528 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004529 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004530 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004531 timestamp_ms += kFrameIntervalMs;
4532 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4533 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004534 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004535 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004536 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004537 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004538 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4539 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004540
Evan Shrubsole64469032020-06-11 10:45:29 +02004541 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4542 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004543 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004544 timestamp_ms += kFrameIntervalMs;
4545 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4546 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004547 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004548 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4549 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004550 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004551 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004552
4553 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004554 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004555 timestamp_ms += kFrameIntervalMs;
4556 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004557 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004558 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004559 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004560 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4561 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004562 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004563 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004564
mflodmancc3d4422017-08-03 08:27:51 -07004565 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004566}
4567
mflodmancc3d4422017-08-03 08:27:51 -07004568TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004569 const int kWidth = 640;
4570 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004571
Henrik Boström381d1092020-05-12 18:49:07 +02004572 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004573 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004574
perkj803d97f2016-11-01 11:45:46 -07004575 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004576 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004577 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004578 }
4579
mflodmancc3d4422017-08-03 08:27:51 -07004580 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004581 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004582 video_source_.IncomingCapturedFrame(CreateFrame(
4583 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004584 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004585 }
4586
mflodmancc3d4422017-08-03 08:27:51 -07004587 video_stream_encoder_->Stop();
4588 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004589 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004590
Ying Wangef3998f2019-12-09 13:06:53 +01004591 EXPECT_METRIC_EQ(
4592 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4593 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004594 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4595}
4596
mflodmancc3d4422017-08-03 08:27:51 -07004597TEST_F(VideoStreamEncoderTest,
4598 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004599 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004600 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004601 const int kWidth = 640;
4602 const int kHeight = 360;
4603
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004604 video_stream_encoder_->SetSource(&video_source_,
4605 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004606
4607 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4608 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004609 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004610 }
4611
mflodmancc3d4422017-08-03 08:27:51 -07004612 video_stream_encoder_->Stop();
4613 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004614 stats_proxy_.reset();
4615
4616 EXPECT_EQ(0,
4617 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4618}
4619
Per Kjellanderdcef6412020-10-07 15:09:05 +02004620TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4621 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004622 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004623 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004624
4625 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004626 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 10:07:11 +02004627 SimulcastRateAllocator(fake_encoder_.config())
4628 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004629 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004630
Henrik Boström381d1092020-05-12 18:49:07 +02004631 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004632 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004633
sprang57c2fff2017-01-16 06:24:02 -08004634 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004635 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4636 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004637 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4638 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4639
Erik Språngd7329ca2019-02-21 21:19:53 +01004640 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004641 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004642 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004643
Per Kjellanderdcef6412020-10-07 15:09:05 +02004644 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004645 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004646 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4647 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004648 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004649 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004650
Per Kjellanderdcef6412020-10-07 15:09:05 +02004651 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004652 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004653 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004654 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004655 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4656 WaitForEncodedFrame(CurrentTimeMs());
4657 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004658 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004659 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004660
mflodmancc3d4422017-08-03 08:27:51 -07004661 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004662}
4663
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004664TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004665 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004666 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004667 kVideoLayersAllocation);
4668
4669 const int kDefaultFps = 30;
4670
4671 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004672 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004673
4674 video_source_.IncomingCapturedFrame(
4675 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4676 WaitForEncodedFrame(CurrentTimeMs());
4677 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4678 VideoLayersAllocation last_layer_allocation =
4679 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004680 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004681 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4682
4683 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004684 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004685 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 10:07:11 +02004686 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 17:53:22 +02004687 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4688
Erik Språng9d69cbe2020-10-22 17:44:42 +02004689 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004690 int number_of_layers_allocation = 1;
4691 const int64_t start_time_ms = CurrentTimeMs();
4692 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4693 video_source_.IncomingCapturedFrame(
4694 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4695 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004696 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4697 number_of_layers_allocation = sink_.number_of_layers_allocations();
4698 VideoLayersAllocation new_allocation =
4699 sink_.GetLastVideoLayersAllocation();
4700 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4701 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4702 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4703 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4704 .target_bitrate_per_temporal_layer,
4705 last_layer_allocation.active_spatial_layers[0]
4706 .target_bitrate_per_temporal_layer);
4707 last_layer_allocation = new_allocation;
4708 }
4709 }
4710 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4711 video_stream_encoder_->Stop();
4712}
4713
4714TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004715 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004716 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4717 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4718 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004719 VideoEncoderConfig video_encoder_config;
4720 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4721 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004722 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004723 video_encoder_config.content_type =
4724 VideoEncoderConfig::ContentType::kRealtimeVideo;
4725 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004726 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004727 VideoEncoder::GetDefaultVp8Settings());
4728 for (auto& layer : video_encoder_config.simulcast_layers) {
4729 layer.num_temporal_layers = 2;
4730 }
4731 // Simulcast layers are used for enabling/disabling streams.
4732 video_encoder_config.simulcast_layers[0].active = true;
4733 video_encoder_config.simulcast_layers[1].active = false;
4734 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004735 ConfigureEncoder(std::move(video_encoder_config),
4736 VideoStreamEncoder::BitrateAllocationCallbackType::
4737 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004738
4739 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004740 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004741
4742 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4743 WaitForEncodedFrame(CurrentTimeMs());
4744 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4745 VideoLayersAllocation last_layer_allocation =
4746 sink_.GetLastVideoLayersAllocation();
4747
4748 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4749 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4750 .target_bitrate_per_temporal_layer,
4751 SizeIs(2));
4752 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4753 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4754 video_stream_encoder_->Stop();
4755}
4756
4757TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004758 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004759 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4760 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4761 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004762 VideoEncoderConfig video_encoder_config;
4763 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4764 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004765 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004766 video_encoder_config.content_type =
4767 VideoEncoderConfig::ContentType::kRealtimeVideo;
4768 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004769 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004770 VideoEncoder::GetDefaultVp8Settings());
4771 for (auto& layer : video_encoder_config.simulcast_layers) {
4772 layer.num_temporal_layers = 2;
4773 }
4774 // Simulcast layers are used for enabling/disabling streams.
4775 video_encoder_config.simulcast_layers[0].active = true;
4776 video_encoder_config.simulcast_layers[1].active = false;
4777 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004778 ConfigureEncoder(std::move(video_encoder_config),
4779 VideoStreamEncoder::BitrateAllocationCallbackType::
4780 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004781
4782 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004783 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004784
4785 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4786 WaitForEncodedFrame(CurrentTimeMs());
4787 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4788 VideoLayersAllocation last_layer_allocation =
4789 sink_.GetLastVideoLayersAllocation();
4790
4791 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4792 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4793 .target_bitrate_per_temporal_layer,
4794 SizeIs(2));
4795 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4796
4797 video_stream_encoder_->Stop();
4798}
4799
4800TEST_F(VideoStreamEncoderTest,
4801 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4802 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4803 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004804 VideoEncoderConfig video_encoder_config;
4805 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4806 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004807 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004808 video_encoder_config.content_type =
4809 VideoEncoderConfig::ContentType::kRealtimeVideo;
4810 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4811 vp9_settings.numberOfSpatialLayers = 2;
4812 vp9_settings.numberOfTemporalLayers = 2;
4813 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4814 vp9_settings.automaticResizeOn = false;
4815 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004816 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004817 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004818 ConfigureEncoder(std::move(video_encoder_config),
4819 VideoStreamEncoder::BitrateAllocationCallbackType::
4820 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004821
4822 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004823 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004824
4825 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4826 WaitForEncodedFrame(CurrentTimeMs());
4827 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4828 VideoLayersAllocation last_layer_allocation =
4829 sink_.GetLastVideoLayersAllocation();
4830
4831 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4832 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4833 .target_bitrate_per_temporal_layer,
4834 SizeIs(2));
4835 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4836 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4837 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4838 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4839 .target_bitrate_per_temporal_layer,
4840 SizeIs(2));
4841 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4842 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4843 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4844
4845 // Since full SVC is used, expect the top layer to utilize the full target
4846 // rate.
4847 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4848 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004849 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004850 video_stream_encoder_->Stop();
4851}
4852
4853TEST_F(VideoStreamEncoderTest,
4854 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4855 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4856 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004857 VideoEncoderConfig video_encoder_config;
4858 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4859 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004860 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004861 video_encoder_config.content_type =
4862 VideoEncoderConfig::ContentType::kRealtimeVideo;
4863 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4864 vp9_settings.numberOfSpatialLayers = 2;
4865 vp9_settings.numberOfTemporalLayers = 2;
4866 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4867 vp9_settings.automaticResizeOn = false;
4868 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004869 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004870 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004871 ConfigureEncoder(std::move(video_encoder_config),
4872 VideoStreamEncoder::BitrateAllocationCallbackType::
4873 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004874
4875 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004876 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004877
4878 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4879 WaitForEncodedFrame(CurrentTimeMs());
4880 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4881 VideoLayersAllocation last_layer_allocation =
4882 sink_.GetLastVideoLayersAllocation();
4883
4884 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4885 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4886 .target_bitrate_per_temporal_layer,
4887 SizeIs(1));
4888 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4889 .target_bitrate_per_temporal_layer,
4890 SizeIs(1));
4891 // Since full SVC is used, expect the top layer to utilize the full target
4892 // rate.
4893 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4894 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004895 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004896 video_stream_encoder_->Stop();
4897}
4898
4899TEST_F(VideoStreamEncoderTest,
4900 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4901 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4902 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004903 VideoEncoderConfig video_encoder_config;
4904 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4905 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004906 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004907 video_encoder_config.content_type =
4908 VideoEncoderConfig::ContentType::kRealtimeVideo;
4909 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4910 vp9_settings.numberOfSpatialLayers = 2;
4911 vp9_settings.numberOfTemporalLayers = 2;
4912 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4913 vp9_settings.automaticResizeOn = false;
4914 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004915 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004916 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004917 ConfigureEncoder(std::move(video_encoder_config),
4918 VideoStreamEncoder::BitrateAllocationCallbackType::
4919 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004920
4921 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004922 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004923
4924 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4925 WaitForEncodedFrame(CurrentTimeMs());
4926 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4927 VideoLayersAllocation last_layer_allocation =
4928 sink_.GetLastVideoLayersAllocation();
4929
4930 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4931 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4932 .target_bitrate_per_temporal_layer,
4933 SizeIs(2));
4934 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4935 .target_bitrate_per_temporal_layer,
4936 SizeIs(2));
4937 // Since KSVC is, spatial layers are independend except on key frames.
4938 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4939 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004940 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004941 video_stream_encoder_->Stop();
4942}
4943
4944TEST_F(VideoStreamEncoderTest,
4945 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4946 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4947 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4948 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004949 VideoEncoderConfig video_encoder_config;
4950 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4951 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004952 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004953 video_encoder_config.content_type =
4954 VideoEncoderConfig::ContentType::kRealtimeVideo;
4955 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4956 vp9_settings.numberOfSpatialLayers = 3;
4957 vp9_settings.numberOfTemporalLayers = 2;
4958 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4959 vp9_settings.automaticResizeOn = false;
4960 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004961 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004962 vp9_settings);
4963 // Simulcast layers are used for enabling/disabling streams.
4964 video_encoder_config.simulcast_layers.resize(3);
4965 video_encoder_config.simulcast_layers[0].active = false;
4966 video_encoder_config.simulcast_layers[1].active = true;
4967 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004968 ConfigureEncoder(std::move(video_encoder_config),
4969 VideoStreamEncoder::BitrateAllocationCallbackType::
4970 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004971
4972 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004973 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004974
4975 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4976 WaitForEncodedFrame(CurrentTimeMs());
4977 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4978 VideoLayersAllocation last_layer_allocation =
4979 sink_.GetLastVideoLayersAllocation();
4980
4981 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4982 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4983 .target_bitrate_per_temporal_layer,
4984 SizeIs(2));
4985 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4986 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4987
4988 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4989 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4990 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4991 .target_bitrate_per_temporal_layer,
4992 SizeIs(2));
4993 // Since full SVC is used, expect the top layer to utilize the full target
4994 // rate.
4995 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4996 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004997 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004998 video_stream_encoder_->Stop();
4999}
5000
5001TEST_F(VideoStreamEncoderTest,
5002 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
5003 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5004 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5005 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005006 VideoEncoderConfig video_encoder_config;
5007 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5008 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005009 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005010 video_encoder_config.content_type =
5011 VideoEncoderConfig::ContentType::kRealtimeVideo;
5012 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5013 vp9_settings.numberOfSpatialLayers = 3;
5014 vp9_settings.numberOfTemporalLayers = 2;
5015 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5016 vp9_settings.automaticResizeOn = false;
5017 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005018 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005019 vp9_settings);
5020 // Simulcast layers are used for enabling/disabling streams.
5021 video_encoder_config.simulcast_layers.resize(3);
5022 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005023 ConfigureEncoder(std::move(video_encoder_config),
5024 VideoStreamEncoder::BitrateAllocationCallbackType::
5025 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005026
5027 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005028 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005029
5030 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5031 WaitForEncodedFrame(CurrentTimeMs());
5032 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5033 VideoLayersAllocation last_layer_allocation =
5034 sink_.GetLastVideoLayersAllocation();
5035
5036 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
5037 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5038 .target_bitrate_per_temporal_layer,
5039 SizeIs(2));
5040 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
5041 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5042
5043 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
5044 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
5045 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
5046 .target_bitrate_per_temporal_layer,
5047 SizeIs(2));
5048 video_stream_encoder_->Stop();
5049}
5050
5051TEST_F(VideoStreamEncoderTest,
5052 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
5053 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5054 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5055 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005056 VideoEncoderConfig video_encoder_config;
5057 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5058 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005059 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005060 video_encoder_config.content_type =
5061 VideoEncoderConfig::ContentType::kRealtimeVideo;
5062 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5063 vp9_settings.numberOfSpatialLayers = 3;
5064 vp9_settings.numberOfTemporalLayers = 2;
5065 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5066 vp9_settings.automaticResizeOn = false;
5067 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005068 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005069 vp9_settings);
5070 // Simulcast layers are used for enabling/disabling streams.
5071 video_encoder_config.simulcast_layers.resize(3);
5072 video_encoder_config.simulcast_layers[0].active = false;
5073 video_encoder_config.simulcast_layers[1].active = false;
5074 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005075 ConfigureEncoder(std::move(video_encoder_config),
5076 VideoStreamEncoder::BitrateAllocationCallbackType::
5077 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005078
5079 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005080 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005081
5082 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5083 WaitForEncodedFrame(CurrentTimeMs());
5084 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5085 VideoLayersAllocation last_layer_allocation =
5086 sink_.GetLastVideoLayersAllocation();
5087
5088 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5089 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5090 .target_bitrate_per_temporal_layer,
5091 SizeIs(2));
5092 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5093 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5094 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5095 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02005096 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005097 video_stream_encoder_->Stop();
5098}
5099
5100TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5101 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005102 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005103 kVideoLayersAllocation);
5104 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005105 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005106
5107 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5108 WaitForEncodedFrame(CurrentTimeMs());
5109 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5110 VideoLayersAllocation last_layer_allocation =
5111 sink_.GetLastVideoLayersAllocation();
5112
5113 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5114 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5115 .target_bitrate_per_temporal_layer,
5116 SizeIs(1));
5117 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5118 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005119 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005120 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5121 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5122 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5123 video_stream_encoder_->Stop();
5124}
5125
5126TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02005127 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5128 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005129 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02005130 kVideoLayersAllocation);
5131
5132 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005133 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005134
5135 video_source_.IncomingCapturedFrame(
5136 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5137 WaitForEncodedFrame(CurrentTimeMs());
5138 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5139 VideoLayersAllocation last_layer_allocation =
5140 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02005141 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02005142 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5143 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5144 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005145 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 17:53:22 +02005146
5147 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005148 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5149 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005150 video_source_.IncomingCapturedFrame(
5151 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5152 WaitForEncodedFrame(CurrentTimeMs());
5153
5154 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5155 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5156 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5157 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5158 .target_bitrate_per_temporal_layer[0],
5159 DataRate::Zero());
5160
5161 video_stream_encoder_->Stop();
5162}
5163
Per Kjellander4190ce92020-12-15 17:24:55 +01005164TEST_F(VideoStreamEncoderTest,
5165 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5166 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005167 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01005168 kVideoLayersAllocation);
5169
5170 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005171 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5172 0, 0, 0);
Per Kjellander4190ce92020-12-15 17:24:55 +01005173
5174 video_source_.IncomingCapturedFrame(
5175 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5176 WaitForEncodedFrame(CurrentTimeMs());
5177 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5178 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5179 SizeIs(2));
5180 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5181 codec_width_);
5182 EXPECT_EQ(
5183 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5184 codec_height_);
5185
5186 video_source_.IncomingCapturedFrame(
5187 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5188 WaitForEncodedFrame(CurrentTimeMs());
5189 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5190 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5191 SizeIs(2));
5192 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5193 codec_width_ / 2);
5194 EXPECT_EQ(
5195 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5196 codec_height_ / 2);
5197
5198 video_stream_encoder_->Stop();
5199}
5200
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005201TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5202 // 2 TLs configured, temporal layers supported by encoder.
5203 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005204 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005205 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005206 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005207 fake_encoder_.SetTemporalLayersSupported(0, true);
5208
5209 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 10:07:11 +02005210 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005211 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005212 kNumTemporalLayers, /*temporal_id*/ 0,
5213 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005214 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005215 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005216 kNumTemporalLayers, /*temporal_id*/ 1,
5217 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005218 VideoBitrateAllocation expected_bitrate;
5219 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5220 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5221
5222 VerifyAllocatedBitrate(expected_bitrate);
5223 video_stream_encoder_->Stop();
5224}
5225
5226TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5227 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005228 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005229 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005230 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005231 fake_encoder_.SetTemporalLayersSupported(0, false);
5232
5233 // Temporal layers not supported by the encoder.
5234 // Total bitrate should be at ti:0.
5235 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005236 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005237
5238 VerifyAllocatedBitrate(expected_bitrate);
5239 video_stream_encoder_->Stop();
5240}
5241
5242TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005243 webrtc::test::ScopedKeyValueConfig field_trials(
5244 field_trials_,
Per Kjellanderdcef6412020-10-07 15:09:05 +02005245 "WebRTC-Video-QualityScalerSettings/"
5246 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5247 // Reset encoder for field trials to take effect.
5248 ConfigureEncoder(video_encoder_config_.Copy());
5249
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005250 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005251 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005252 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005253 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005254 fake_encoder_.SetTemporalLayersSupported(0, true);
5255 fake_encoder_.SetTemporalLayersSupported(1, false);
5256
5257 const int kS0Bps = 150000;
5258 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005259 kS0Bps *
5260 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5261 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005262 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005263 kS0Bps *
5264 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5265 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005266 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005267 // Temporal layers not supported by si:1.
5268 VideoBitrateAllocation expected_bitrate;
5269 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5270 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5271 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5272
5273 VerifyAllocatedBitrate(expected_bitrate);
5274 video_stream_encoder_->Stop();
5275}
5276
Niels Möller7dc26b72017-12-06 10:27:48 +01005277TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5278 const int kFrameWidth = 1280;
5279 const int kFrameHeight = 720;
5280 const int kFramerate = 24;
5281
Henrik Boström381d1092020-05-12 18:49:07 +02005282 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005283 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005284 test::FrameForwarder source;
5285 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005286 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005287
5288 // Insert a single frame, triggering initial configuration.
5289 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5290 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5291
5292 EXPECT_EQ(
5293 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5294 kDefaultFramerate);
5295
5296 // Trigger reconfigure encoder (without resetting the entire instance).
5297 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005298 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5299 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005300 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005301 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005302 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005303 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5304
5305 // Detector should be updated with fps limit from codec config.
5306 EXPECT_EQ(
5307 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5308 kFramerate);
5309
5310 // Trigger overuse, max framerate should be reduced.
5311 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5312 stats.input_frame_rate = kFramerate;
5313 stats_proxy_->SetMockStats(stats);
5314 video_stream_encoder_->TriggerCpuOveruse();
5315 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5316 int adapted_framerate =
5317 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5318 EXPECT_LT(adapted_framerate, kFramerate);
5319
5320 // Trigger underuse, max framerate should go back to codec configured fps.
5321 // Set extra low fps, to make sure it's actually reset, not just incremented.
5322 stats = stats_proxy_->GetStats();
5323 stats.input_frame_rate = adapted_framerate / 2;
5324 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005325 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005326 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5327 EXPECT_EQ(
5328 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5329 kFramerate);
5330
5331 video_stream_encoder_->Stop();
5332}
5333
5334TEST_F(VideoStreamEncoderTest,
5335 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5336 const int kFrameWidth = 1280;
5337 const int kFrameHeight = 720;
5338 const int kLowFramerate = 15;
5339 const int kHighFramerate = 25;
5340
Henrik Boström381d1092020-05-12 18:49:07 +02005341 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005342 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005343 test::FrameForwarder source;
5344 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005345 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005346
5347 // Trigger initial configuration.
5348 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005349 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5350 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005351 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005352 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005353 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005354 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005355 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5356
5357 EXPECT_EQ(
5358 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5359 kLowFramerate);
5360
5361 // Trigger overuse, max framerate should be reduced.
5362 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5363 stats.input_frame_rate = kLowFramerate;
5364 stats_proxy_->SetMockStats(stats);
5365 video_stream_encoder_->TriggerCpuOveruse();
5366 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5367 int adapted_framerate =
5368 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5369 EXPECT_LT(adapted_framerate, kLowFramerate);
5370
5371 // Reconfigure the encoder with a new (higher max framerate), max fps should
5372 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005373 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005374 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5375 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005376 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005377 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5378
5379 EXPECT_EQ(
5380 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5381 adapted_framerate);
5382
5383 // Trigger underuse, max framerate should go back to codec configured fps.
5384 stats = stats_proxy_->GetStats();
5385 stats.input_frame_rate = adapted_framerate;
5386 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005387 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005388 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5389 EXPECT_EQ(
5390 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5391 kHighFramerate);
5392
5393 video_stream_encoder_->Stop();
5394}
5395
mflodmancc3d4422017-08-03 08:27:51 -07005396TEST_F(VideoStreamEncoderTest,
5397 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005398 const int kFrameWidth = 1280;
5399 const int kFrameHeight = 720;
5400 const int kFramerate = 24;
5401
Henrik Boström381d1092020-05-12 18:49:07 +02005402 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005403 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005404 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005405 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005406 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005407
5408 // Trigger initial configuration.
5409 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005410 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5411 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005412 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 04:21:07 -07005413 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005414 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005415 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005416 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005417
Niels Möller7dc26b72017-12-06 10:27:48 +01005418 EXPECT_EQ(
5419 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5420 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005421
5422 // Trigger overuse, max framerate should be reduced.
5423 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5424 stats.input_frame_rate = kFramerate;
5425 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005426 video_stream_encoder_->TriggerCpuOveruse();
5427 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005428 int adapted_framerate =
5429 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005430 EXPECT_LT(adapted_framerate, kFramerate);
5431
5432 // Change degradation preference to not enable framerate scaling. Target
5433 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005434 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005435 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005436 EXPECT_EQ(
5437 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5438 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005439
mflodmancc3d4422017-08-03 08:27:51 -07005440 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005441}
5442
mflodmancc3d4422017-08-03 08:27:51 -07005443TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005444 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005445 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005446 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5447 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5448 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005449 const int kWidth = 640;
5450 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005451
asaperssonfab67072017-04-04 05:51:49 -07005452 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005453
5454 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005455 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005456
5457 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005458 EXPECT_TRUE_WAIT(
5459 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005460
sprangc5d62e22017-04-02 23:53:04 -07005461 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005462
asaperssonfab67072017-04-04 05:51:49 -07005463 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005464 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005465 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005466
5467 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005468 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005469
Henrik Boström2671dac2020-05-19 16:29:09 +02005470 EXPECT_TRUE_WAIT(
5471 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005472
mflodmancc3d4422017-08-03 08:27:51 -07005473 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005474}
5475
mflodmancc3d4422017-08-03 08:27:51 -07005476TEST_F(VideoStreamEncoderTest,
5477 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005478 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005479 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005480 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5481 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5482 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005483 const int kWidth = 640;
5484 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005485
5486 // We expect the n initial frames to get dropped.
5487 int i;
5488 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005489 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005490 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005491 }
5492 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005493 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005494 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005495
5496 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005497 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005498
mflodmancc3d4422017-08-03 08:27:51 -07005499 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005500}
5501
mflodmancc3d4422017-08-03 08:27:51 -07005502TEST_F(VideoStreamEncoderTest,
5503 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005504 const int kWidth = 640;
5505 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005506 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005507 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005508
5509 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005510 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005511 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005512
asaperssonfab67072017-04-04 05:51:49 -07005513 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005514 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005515 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005516
mflodmancc3d4422017-08-03 08:27:51 -07005517 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005518}
5519
mflodmancc3d4422017-08-03 08:27:51 -07005520TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005521 const int kWidth = 640;
5522 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005523 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005524
5525 VideoEncoderConfig video_encoder_config;
5526 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5527 // Make format different, to force recreation of encoder.
5528 video_encoder_config.video_format.parameters["foo"] = "foo";
5529 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005530 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005531 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005532 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005533
kthelgasonb83797b2017-02-14 11:57:25 -08005534 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005535 video_stream_encoder_->SetSource(&video_source_,
5536 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005537
asaperssonfab67072017-04-04 05:51:49 -07005538 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005539 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005540 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005541
mflodmancc3d4422017-08-03 08:27:51 -07005542 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005543 fake_encoder_.SetQualityScaling(true);
5544}
5545
Åsa Persson139f4dc2019-08-02 09:29:58 +02005546TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005547 webrtc::test::ScopedKeyValueConfig field_trials(
5548 field_trials_,
Åsa Persson139f4dc2019-08-02 09:29:58 +02005549 "WebRTC-Video-QualityScalerSettings/"
5550 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5551 // Reset encoder for field trials to take effect.
5552 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005553 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5554 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 09:29:58 +02005555 const int kWidth = 640;
5556 const int kHeight = 360;
5557
Henrik Boström381d1092020-05-12 18:49:07 +02005558 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005559 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005560 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5561 // Frame should not be dropped.
5562 WaitForEncodedFrame(1);
5563
Henrik Boström381d1092020-05-12 18:49:07 +02005564 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005565 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5566 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5567 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005568 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5569 // Frame should not be dropped.
5570 WaitForEncodedFrame(2);
5571
Henrik Boström381d1092020-05-12 18:49:07 +02005572 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005573 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5574 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5575 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005576 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5577 // Expect to drop this frame, the wait should time out.
5578 ExpectDroppedFrame();
5579
5580 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005581 EXPECT_TRUE_WAIT(
5582 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005583 video_stream_encoder_->Stop();
5584}
5585
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005586TEST_F(VideoStreamEncoderTest,
5587 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005588 webrtc::test::ScopedKeyValueConfig field_trials(
5589 field_trials_,
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005590 "WebRTC-Video-QualityScalerSettings/"
5591 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5592 fake_encoder_.SetQualityScaling(false);
5593 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005594 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5595 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005596 const int kWidth = 640;
5597 const int kHeight = 360;
5598
5599 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005600 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005601 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5602 // Frame should not be dropped.
5603 WaitForEncodedFrame(1);
5604
5605 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5606 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5607 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5608 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5609 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5610 // Frame should not be dropped.
5611 WaitForEncodedFrame(2);
5612
5613 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5614 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5615 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5616 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5617 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5618 // Not dropped since quality scaling is disabled.
5619 WaitForEncodedFrame(3);
5620
5621 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005622 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005623 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5624
5625 video_stream_encoder_->Stop();
5626}
5627
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005628TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005629 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005630 // Set simulcast.
5631 ResetEncoder("VP8", 3, 1, 1, false);
5632 fake_encoder_.SetQualityScaling(true);
5633 const int kWidth = 1280;
5634 const int kHeight = 720;
5635 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005636 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005637 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5638 // Frame should not be dropped.
5639 WaitForEncodedFrame(1);
5640
5641 // Trigger QVGA "singlecast"
5642 // Update the config.
5643 VideoEncoderConfig video_encoder_config;
5644 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5645 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005646 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005647 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005648 "VP8", /*max qp*/ 56, /*screencast*/ false,
5649 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005650 for (auto& layer : video_encoder_config.simulcast_layers) {
5651 layer.num_temporal_layers = 1;
5652 layer.max_framerate = kDefaultFramerate;
5653 }
Asa Persson606d3cb2021-10-04 10:07:11 +02005654 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005655 video_encoder_config.content_type =
5656 VideoEncoderConfig::ContentType::kRealtimeVideo;
5657
5658 video_encoder_config.simulcast_layers[0].active = true;
5659 video_encoder_config.simulcast_layers[1].active = false;
5660 video_encoder_config.simulcast_layers[2].active = false;
5661
5662 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5663 kMaxPayloadLength);
5664 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5665
5666 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5667 // Frame should not be dropped.
5668 WaitForEncodedFrame(2);
5669
5670 // Trigger HD "singlecast"
5671 video_encoder_config.simulcast_layers[0].active = false;
5672 video_encoder_config.simulcast_layers[1].active = false;
5673 video_encoder_config.simulcast_layers[2].active = true;
5674
5675 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5676 kMaxPayloadLength);
5677 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5678
5679 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5680 // Frame should be dropped because of initial frame drop.
5681 ExpectDroppedFrame();
5682
5683 // Expect the sink_wants to specify a scaled frame.
5684 EXPECT_TRUE_WAIT(
5685 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5686 video_stream_encoder_->Stop();
5687}
5688
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005689TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005690 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005691 // Set simulcast.
5692 ResetEncoder("VP9", 1, 1, 3, false);
5693 fake_encoder_.SetQualityScaling(true);
5694 const int kWidth = 1280;
5695 const int kHeight = 720;
5696 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005697 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005698 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5699 // Frame should not be dropped.
5700 WaitForEncodedFrame(1);
5701
5702 // Trigger QVGA "singlecast"
5703 // Update the config.
5704 VideoEncoderConfig video_encoder_config;
5705 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5706 &video_encoder_config);
5707 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5708 vp9_settings.numberOfSpatialLayers = 3;
5709 // Since only one layer is active - automatic resize should be enabled.
5710 vp9_settings.automaticResizeOn = true;
5711 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005712 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005713 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005714 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005715 video_encoder_config.content_type =
5716 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 16:22:11 +02005717 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005718 // which SVC layers are active.
5719 video_encoder_config.simulcast_layers.resize(3);
5720
5721 video_encoder_config.simulcast_layers[0].active = true;
5722 video_encoder_config.simulcast_layers[1].active = false;
5723 video_encoder_config.simulcast_layers[2].active = false;
5724
5725 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5726 kMaxPayloadLength);
5727 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5728
5729 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5730 // Frame should not be dropped.
5731 WaitForEncodedFrame(2);
5732
5733 // Trigger HD "singlecast"
5734 video_encoder_config.simulcast_layers[0].active = false;
5735 video_encoder_config.simulcast_layers[1].active = false;
5736 video_encoder_config.simulcast_layers[2].active = true;
5737
5738 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5739 kMaxPayloadLength);
5740 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5741
5742 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5743 // Frame should be dropped because of initial frame drop.
5744 ExpectDroppedFrame();
5745
5746 // Expect the sink_wants to specify a scaled frame.
5747 EXPECT_TRUE_WAIT(
5748 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5749 video_stream_encoder_->Stop();
5750}
5751
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005752TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005753 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5754 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5755 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5756 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5757 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5758 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5759 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5760 fake_encoder_.SetResolutionBitrateLimits(
5761 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5762
5763 VideoEncoderConfig video_encoder_config;
5764 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5765 &video_encoder_config);
5766 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5767 vp9_settings.numberOfSpatialLayers = 3;
5768 // Since only one layer is active - automatic resize should be enabled.
5769 vp9_settings.automaticResizeOn = true;
5770 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005771 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005772 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005773 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005774 video_encoder_config.content_type =
5775 VideoEncoderConfig::ContentType::kRealtimeVideo;
5776 // Simulcast layers are used to indicate which spatial layers are active.
5777 video_encoder_config.simulcast_layers.resize(3);
5778 video_encoder_config.simulcast_layers[0].active = false;
5779 video_encoder_config.simulcast_layers[1].active = true;
5780 video_encoder_config.simulcast_layers[2].active = false;
5781
5782 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5783 kMaxPayloadLength);
5784 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5785
5786 // The encoder bitrate limits for 360p should be used.
5787 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5788 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005789 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5790 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5791 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5792 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5793 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5794 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005795 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005796 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005797 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005798 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005799
5800 // The encoder bitrate limits for 270p should be used.
5801 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5802 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005803 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5804 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5805 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5806 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5807 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5808 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005809 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005810 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005811 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005812 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005813
5814 video_stream_encoder_->Stop();
5815}
5816
5817TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005818 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5819 VideoEncoderConfig video_encoder_config;
5820 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5821 &video_encoder_config);
5822 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5823 vp9_settings.numberOfSpatialLayers = 3;
5824 // Since only one layer is active - automatic resize should be enabled.
5825 vp9_settings.automaticResizeOn = true;
5826 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005827 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005828 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005829 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005830 video_encoder_config.content_type =
5831 VideoEncoderConfig::ContentType::kRealtimeVideo;
5832 // Simulcast layers are used to indicate which spatial layers are active.
5833 video_encoder_config.simulcast_layers.resize(3);
5834 video_encoder_config.simulcast_layers[0].active = false;
5835 video_encoder_config.simulcast_layers[1].active = true;
5836 video_encoder_config.simulcast_layers[2].active = false;
5837
5838 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5839 kMaxPayloadLength);
5840 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5841
5842 // The default bitrate limits for 360p should be used.
5843 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005844 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5845 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005846 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5847 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005848 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5849 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5850 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5851 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5852 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5853 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005854 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005855 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005856 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005857 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005858
5859 // The default bitrate limits for 270p should be used.
5860 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005861 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5862 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005863 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5864 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005865 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5866 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5867 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5868 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5869 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5870 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005871 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005872 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005873 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005874 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005875
5876 video_stream_encoder_->Stop();
5877}
5878
5879TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005880 webrtc::test::ScopedKeyValueConfig field_trials(
5881 field_trials_, "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
Åsa Persson258e9892021-02-25 10:39:51 +01005882 VideoEncoderConfig video_encoder_config;
5883 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5884 &video_encoder_config);
5885 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5886 vp9_settings.numberOfSpatialLayers = 3;
5887 // Since only one layer is active - automatic resize should be enabled.
5888 vp9_settings.automaticResizeOn = true;
5889 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005890 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005891 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005892 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005893 video_encoder_config.content_type =
5894 VideoEncoderConfig::ContentType::kRealtimeVideo;
5895 // Simulcast layers are used to indicate which spatial layers are active.
5896 video_encoder_config.simulcast_layers.resize(3);
5897 video_encoder_config.simulcast_layers[0].active = false;
5898 video_encoder_config.simulcast_layers[1].active = true;
5899 video_encoder_config.simulcast_layers[2].active = false;
5900
5901 // Reset encoder for field trials to take effect.
5902 ConfigureEncoder(video_encoder_config.Copy());
5903
5904 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5905 kMaxPayloadLength);
5906 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5907
5908 // The default bitrate limits for 360p should not be used.
5909 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005910 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5911 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005912 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5913 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005914 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5915 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5916 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5917 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5918 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5919 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005920 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005921 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005922
5923 video_stream_encoder_->Stop();
5924}
5925
5926TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5927 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5928 /*num_spatial_layers=*/1, /*screenshare=*/false);
5929
5930 // The default singlecast bitrate limits for 720p should not be used.
5931 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005932 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5933 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005934 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5935 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005936 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5937 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5938 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5939 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5940 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5941 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005942 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005943 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005944
5945 video_stream_encoder_->Stop();
5946}
5947
5948TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005949 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5950 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5951 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5952 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5953 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5954 fake_encoder_.SetResolutionBitrateLimits(
5955 {kEncoderLimits180p, kEncoderLimits720p});
5956
5957 VideoEncoderConfig video_encoder_config;
5958 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5959 &video_encoder_config);
5960 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5961 vp9_settings.numberOfSpatialLayers = 3;
5962 // Since only one layer is active - automatic resize should be enabled.
5963 vp9_settings.automaticResizeOn = true;
5964 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005965 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005966 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005967 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005968 video_encoder_config.content_type =
5969 VideoEncoderConfig::ContentType::kRealtimeVideo;
5970 // Simulcast layers are used to indicate which spatial layers are active.
5971 video_encoder_config.simulcast_layers.resize(3);
5972 video_encoder_config.simulcast_layers[0].active = true;
5973 video_encoder_config.simulcast_layers[1].active = false;
5974 video_encoder_config.simulcast_layers[2].active = false;
5975
5976 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5977 kMaxPayloadLength);
5978 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5979
5980 // Limits not applied on lowest stream, limits for 180p should not be used.
5981 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5982 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005983 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5984 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5985 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5986 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5987 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5988 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005989 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005990 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005991 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005992 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005993
5994 video_stream_encoder_->Stop();
5995}
5996
5997TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005998 InitialFrameDropActivatesWhenResolutionIncreases) {
5999 const int kWidth = 640;
6000 const int kHeight = 360;
6001
6002 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006003 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006004 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
6005 // Frame should not be dropped.
6006 WaitForEncodedFrame(1);
6007
6008 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006009 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006010 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
6011 // Frame should not be dropped, bitrate not too low for frame.
6012 WaitForEncodedFrame(2);
6013
6014 // Incoming resolution increases.
6015 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
6016 // Expect to drop this frame, bitrate too low for frame.
6017 ExpectDroppedFrame();
6018
6019 // Expect the sink_wants to specify a scaled frame.
6020 EXPECT_TRUE_WAIT(
6021 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
6022 video_stream_encoder_->Stop();
6023}
6024
6025TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
6026 const int kWidth = 640;
6027 const int kHeight = 360;
6028 // So that quality scaling doesn't happen by itself.
6029 fake_encoder_.SetQp(kQpHigh);
6030
6031 AdaptingFrameForwarder source(&time_controller_);
6032 source.set_adaptation_enabled(true);
6033 video_stream_encoder_->SetSource(
6034 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
6035
6036 int timestamp = 1;
6037
6038 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006039 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006040 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6041 WaitForEncodedFrame(timestamp);
6042 timestamp += 9000;
6043 // Long pause to disable all first BWE drop logic.
6044 AdvanceTime(TimeDelta::Millis(1000));
6045
6046 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006047 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006048 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6049 // Not dropped frame, as initial frame drop is disabled by now.
6050 WaitForEncodedFrame(timestamp);
6051 timestamp += 9000;
6052 AdvanceTime(TimeDelta::Millis(100));
6053
6054 // Quality adaptation down.
6055 video_stream_encoder_->TriggerQualityLow();
6056
6057 // Adaptation has an effect.
6058 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6059 5000);
6060
6061 // Frame isn't dropped as initial frame dropper is disabled.
6062 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6063 WaitForEncodedFrame(timestamp);
6064 timestamp += 9000;
6065 AdvanceTime(TimeDelta::Millis(100));
6066
6067 // Quality adaptation up.
6068 video_stream_encoder_->TriggerQualityHigh();
6069
6070 // Adaptation has an effect.
6071 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
6072 5000);
6073
6074 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6075 // Frame should not be dropped, as initial framedropper is off.
6076 WaitForEncodedFrame(timestamp);
6077
6078 video_stream_encoder_->Stop();
6079}
6080
Åsa Persson7f354f82021-02-04 15:52:15 +01006081TEST_F(VideoStreamEncoderTest,
6082 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
6083 const int kMinStartBps360p = 222000;
6084 fake_encoder_.SetResolutionBitrateLimits(
6085 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6086 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6087 800000)});
6088
6089 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6090 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6091 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6092 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
6093 0, 0, 0);
6094 // Frame should not be dropped, bitrate not too low for frame.
6095 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6096 WaitForEncodedFrame(1);
6097
6098 // Incoming resolution increases, initial frame drop activates.
6099 // Frame should be dropped, link allocation too low for frame.
6100 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6101 ExpectDroppedFrame();
6102
6103 // Expect sink_wants to specify a scaled frame.
6104 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6105 5000);
6106 video_stream_encoder_->Stop();
6107}
6108
6109TEST_F(VideoStreamEncoderTest,
6110 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6111 const int kMinStartBps360p = 222000;
6112 fake_encoder_.SetResolutionBitrateLimits(
6113 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6114 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6115 800000)});
6116
6117 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6118 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6119 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6120 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6121 0, 0, 0);
6122 // Frame should not be dropped, bitrate not too low for frame.
6123 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6124 WaitForEncodedFrame(1);
6125
6126 // Incoming resolution increases, initial frame drop activates.
6127 // Frame should be dropped, link allocation not too low for frame.
6128 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6129 WaitForEncodedFrame(2);
6130
6131 video_stream_encoder_->Stop();
6132}
6133
Åsa Perssone644a032019-11-08 15:56:00 +01006134TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006135 webrtc::test::ScopedKeyValueConfig field_trials(
6136 field_trials_,
Åsa Persson06defc42021-09-10 15:28:48 +02006137 "WebRTC-Video-QualityRampupSettings/"
6138 "min_pixels:921600,min_duration_ms:2000/");
6139
6140 const int kWidth = 1280;
6141 const int kHeight = 720;
6142 const int kFps = 10;
6143 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006144
6145 // Reset encoder for field trials to take effect.
6146 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 10:07:11 +02006147 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 11:34:32 +02006148 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01006149 ConfigureEncoder(std::move(config));
6150 fake_encoder_.SetQp(kQpLow);
6151
6152 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006153 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01006154 source.set_adaptation_enabled(true);
6155 video_stream_encoder_->SetSource(&source,
6156 DegradationPreference::MAINTAIN_FRAMERATE);
6157
6158 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 10:07:11 +02006159 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 18:49:07 +02006160 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006161 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006162
6163 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 15:28:48 +02006164 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006165 int64_t timestamp_ms = kFrameIntervalMs;
6166 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6167 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02006168 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6169 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01006170
6171 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02006172 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6173 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006174
Artem Titovab30d722021-07-27 16:22:11 +02006175 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 15:28:48 +02006176 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 15:56:00 +01006177 for (size_t i = 1; i <= 10; i++) {
6178 timestamp_ms += kFrameIntervalMs;
6179 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6180 WaitForEncodedFrame(timestamp_ms);
6181 }
Åsa Persson06defc42021-09-10 15:28:48 +02006182
6183 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6184 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6185 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6186 timestamp_ms += kFrameIntervalMs;
6187 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6188 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 15:56:00 +01006189 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6190 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6191
Åsa Persson06defc42021-09-10 15:28:48 +02006192 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 15:56:00 +01006193 timestamp_ms += kFrameIntervalMs;
6194 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6195 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006196 // The ramp-up code involves the adaptation queue, give it time to execute.
6197 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006198 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006199 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006200
6201 // Frame should not be adapted.
6202 timestamp_ms += kFrameIntervalMs;
6203 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6204 WaitForEncodedFrame(kWidth, kHeight);
6205 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6206
6207 video_stream_encoder_->Stop();
6208}
6209
mflodmancc3d4422017-08-03 08:27:51 -07006210TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006211 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006212 webrtc::test::ScopedKeyValueConfig field_trials(
6213 field_trials_, "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006214 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006215 source.set_adaptation_enabled(true);
6216 video_stream_encoder_->SetSource(&source,
6217 DegradationPreference::MAINTAIN_FRAMERATE);
6218 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006219 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006220 fake_encoder_.SetQp(kQpHigh + 1);
6221 const int kWidth = 1280;
6222 const int kHeight = 720;
6223 const int64_t kFrameIntervalMs = 100;
6224 int64_t timestamp_ms = kFrameIntervalMs;
6225 for (size_t i = 1; i <= 100; i++) {
6226 timestamp_ms += kFrameIntervalMs;
6227 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6228 WaitForEncodedFrame(timestamp_ms);
6229 }
6230 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6231 // for the first time.
6232 // TODO(eshr): We should avoid these waits by using threads with simulated
6233 // time.
6234 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6235 2000 * 2.5 * 2);
6236 timestamp_ms += kFrameIntervalMs;
6237 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6238 WaitForEncodedFrame(timestamp_ms);
6239 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6240 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6241 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6242
6243 // Disable Quality scaling by turning off scaler on the encoder and
6244 // reconfiguring.
6245 fake_encoder_.SetQualityScaling(false);
6246 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6247 kMaxPayloadLength);
6248 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Markus Handell28c71802021-11-08 10:11:55 +01006249 AdvanceTime(TimeDelta::Zero());
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006250 // Since we turned off the quality scaler, the adaptations made by it are
6251 // removed.
6252 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6253 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6254
6255 video_stream_encoder_->Stop();
6256}
6257
6258TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006259 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6260 const int kTooSmallWidth = 10;
6261 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006262 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006263 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006264
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006265 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006266 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006267 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006268 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006269 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006270 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6271
6272 // Trigger adapt down, too small frame, expect no change.
6273 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006274 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006275 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006276 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006277 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6278 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6279
mflodmancc3d4422017-08-03 08:27:51 -07006280 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006281}
6282
mflodmancc3d4422017-08-03 08:27:51 -07006283TEST_F(VideoStreamEncoderTest,
6284 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006285 const int kTooSmallWidth = 10;
6286 const int kTooSmallHeight = 10;
6287 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006288 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006289 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006290
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006291 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006292 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006293 video_stream_encoder_->SetSource(&source,
6294 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006295 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006296 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6297 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6298
6299 // Trigger adapt down, expect limited framerate.
6300 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006301 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006302 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006303 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006304 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6305 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6306 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6307
6308 // Trigger adapt down, too small frame, expect no change.
6309 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006310 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006311 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006312 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006313 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6314 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6315 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6316
mflodmancc3d4422017-08-03 08:27:51 -07006317 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006318}
6319
mflodmancc3d4422017-08-03 08:27:51 -07006320TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006321 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006322 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006323 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006324 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006325 const int kFrameWidth = 1280;
6326 const int kFrameHeight = 720;
6327 video_source_.IncomingCapturedFrame(
6328 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006329 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006330 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006331}
6332
sprangb1ca0732017-02-01 08:38:12 -08006333// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006334TEST_F(VideoStreamEncoderTest,
6335 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006336 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006337 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006338
6339 const int kFrameWidth = 1280;
6340 const int kFrameHeight = 720;
6341 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006342 // requested by
6343 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006344 video_source_.set_adaptation_enabled(true);
6345
6346 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006347 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006348 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006349
6350 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006351 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006352 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006353 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006354 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006355
asaperssonfab67072017-04-04 05:51:49 -07006356 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006357 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006358 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006359 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006360 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006361
mflodmancc3d4422017-08-03 08:27:51 -07006362 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006363}
sprangfe627f32017-03-29 08:24:59 -07006364
mflodmancc3d4422017-08-03 08:27:51 -07006365TEST_F(VideoStreamEncoderTest,
6366 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006367 const int kFrameWidth = 1280;
6368 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006369
Henrik Boström381d1092020-05-12 18:49:07 +02006370 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006371 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006372 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006373 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006374 video_source_.set_adaptation_enabled(true);
6375
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006376 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006377
6378 video_source_.IncomingCapturedFrame(
6379 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006380 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006381
6382 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006383 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006384
6385 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006386 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006387 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006388 video_source_.IncomingCapturedFrame(
6389 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006390 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006391 }
6392
6393 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006394 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006395 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006396 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006397 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006398 video_source_.IncomingCapturedFrame(
6399 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006400 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006401 ++num_frames_dropped;
6402 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006403 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006404 }
6405 }
6406
sprang4847ae62017-06-27 07:06:52 -07006407 // Add some slack to account for frames dropped by the frame dropper.
6408 const int kErrorMargin = 1;
6409 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006410 kErrorMargin);
6411
6412 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006413 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006414 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006415 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006416 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006417 video_source_.IncomingCapturedFrame(
6418 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006419 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006420 ++num_frames_dropped;
6421 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006422 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006423 }
6424 }
sprang4847ae62017-06-27 07:06:52 -07006425 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006426 kErrorMargin);
6427
6428 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006429 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006430 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006431 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006432 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006433 video_source_.IncomingCapturedFrame(
6434 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006435 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006436 ++num_frames_dropped;
6437 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006438 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006439 }
6440 }
sprang4847ae62017-06-27 07:06:52 -07006441 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006442 kErrorMargin);
6443
6444 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006445 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006446 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006447 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006448 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006449 video_source_.IncomingCapturedFrame(
6450 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006451 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006452 ++num_frames_dropped;
6453 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006454 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006455 }
6456 }
6457 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6458
mflodmancc3d4422017-08-03 08:27:51 -07006459 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006460}
6461
mflodmancc3d4422017-08-03 08:27:51 -07006462TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006463 const int kFramerateFps = 5;
6464 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006465 const int kFrameWidth = 1280;
6466 const int kFrameHeight = 720;
6467
sprang4847ae62017-06-27 07:06:52 -07006468 // Reconfigure encoder with two temporal layers and screensharing, which will
6469 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006470 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006471
Henrik Boström381d1092020-05-12 18:49:07 +02006472 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006473 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006474 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006475 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006476 video_source_.set_adaptation_enabled(true);
6477
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006478 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006479
6480 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006481 rtc::VideoSinkWants last_wants;
6482 do {
6483 last_wants = video_source_.sink_wants();
6484
sprangc5d62e22017-04-02 23:53:04 -07006485 // Insert frames to get a new fps estimate...
6486 for (int j = 0; j < kFramerateFps; ++j) {
6487 video_source_.IncomingCapturedFrame(
6488 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006489 if (video_source_.last_sent_width()) {
6490 sink_.WaitForEncodedFrame(timestamp_ms);
6491 }
sprangc5d62e22017-04-02 23:53:04 -07006492 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006493 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006494 }
6495 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006496 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006497 } while (video_source_.sink_wants().max_framerate_fps <
6498 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006499
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006500 EXPECT_THAT(video_source_.sink_wants(),
6501 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006502
mflodmancc3d4422017-08-03 08:27:51 -07006503 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006504}
asaperssonf7e294d2017-06-13 23:25:22 -07006505
mflodmancc3d4422017-08-03 08:27:51 -07006506TEST_F(VideoStreamEncoderTest,
6507 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006508 const int kWidth = 1280;
6509 const int kHeight = 720;
6510 const int64_t kFrameIntervalMs = 150;
6511 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006512 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006513 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006514
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006515 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006516 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006517 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006518 video_stream_encoder_->SetSource(&source,
6519 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006520 timestamp_ms += kFrameIntervalMs;
6521 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006522 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006523 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006524 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6525 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6526 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6527
6528 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006529 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006530 timestamp_ms += kFrameIntervalMs;
6531 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006532 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006533 EXPECT_THAT(source.sink_wants(),
6534 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006535 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6536 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6537 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6538
6539 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006540 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006541 timestamp_ms += kFrameIntervalMs;
6542 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006543 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006544 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006545 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6546 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6547 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6548
6549 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006550 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006551 timestamp_ms += kFrameIntervalMs;
6552 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006553 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006554 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006555 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6556 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6557 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6558
6559 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006560 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006561 timestamp_ms += kFrameIntervalMs;
6562 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006563 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006564 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006565 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6566 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6567 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6568
6569 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006570 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006571 timestamp_ms += kFrameIntervalMs;
6572 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006573 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006574 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006575 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6576 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6577 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6578
6579 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006580 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006581 timestamp_ms += kFrameIntervalMs;
6582 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006583 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006584 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006585 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6586 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6587 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6588
6589 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006590 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006591 timestamp_ms += kFrameIntervalMs;
6592 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006593 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006594 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006595 rtc::VideoSinkWants last_wants = source.sink_wants();
6596 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6597 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6598 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6599
6600 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006601 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006602 timestamp_ms += kFrameIntervalMs;
6603 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006604 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006605 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006606 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6607 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6608 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6609
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006610 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006611 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006612 timestamp_ms += kFrameIntervalMs;
6613 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006614 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006615 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006616 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6617 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6618 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6619
6620 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006621 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006622 timestamp_ms += kFrameIntervalMs;
6623 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006624 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006625 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006626 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6627 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6628 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6629
6630 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006631 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006632 timestamp_ms += kFrameIntervalMs;
6633 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006634 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006635 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006636 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6637 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6638 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6639
6640 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006641 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006642 timestamp_ms += kFrameIntervalMs;
6643 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006644 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006645 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006646 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6647 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6648 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6649
6650 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006651 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006652 timestamp_ms += kFrameIntervalMs;
6653 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006654 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006655 EXPECT_THAT(source.sink_wants(), FpsMax());
6656 EXPECT_EQ(source.sink_wants().max_pixel_count,
6657 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006658 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6659 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6660 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6661
6662 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006663 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006664 timestamp_ms += kFrameIntervalMs;
6665 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006666 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006667 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006668 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6669 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6670 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6671
Åsa Persson30ab0152019-08-27 12:22:33 +02006672 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006673 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006674 timestamp_ms += kFrameIntervalMs;
6675 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006676 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006677 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006678 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006679 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6680 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6681 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6682
6683 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006684 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006685 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006686 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6687
mflodmancc3d4422017-08-03 08:27:51 -07006688 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006689}
6690
mflodmancc3d4422017-08-03 08:27:51 -07006691TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006692 const int kWidth = 1280;
6693 const int kHeight = 720;
6694 const int64_t kFrameIntervalMs = 150;
6695 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006696 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006697 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006698
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006699 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006700 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006701 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006702 video_stream_encoder_->SetSource(&source,
6703 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006704 timestamp_ms += kFrameIntervalMs;
6705 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006706 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006707 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006708 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6709 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6710 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6711 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6712 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6713 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6714
6715 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006716 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006717 timestamp_ms += kFrameIntervalMs;
6718 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006719 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006720 EXPECT_THAT(source.sink_wants(),
6721 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006722 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6723 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6724 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6725 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6726 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6727 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6728
6729 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006730 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006731 timestamp_ms += kFrameIntervalMs;
6732 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006733 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006734 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006735 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6736 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6737 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6738 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6739 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6740 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6741
6742 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006743 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006744 timestamp_ms += kFrameIntervalMs;
6745 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006746 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006747 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006748 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006749 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6750 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6751 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6752 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6753 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6754
Evan Shrubsole64469032020-06-11 10:45:29 +02006755 // Trigger cpu adapt up, expect no change since QP is most limited.
6756 {
6757 // Store current sink wants since we expect no change and if there is no
6758 // change then last_wants() is not updated.
6759 auto previous_sink_wants = source.sink_wants();
6760 video_stream_encoder_->TriggerCpuUnderuse();
6761 timestamp_ms += kFrameIntervalMs;
6762 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6763 WaitForEncodedFrame(timestamp_ms);
6764 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6765 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6766 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6767 }
6768
6769 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6770 video_stream_encoder_->TriggerQualityHigh();
6771 timestamp_ms += kFrameIntervalMs;
6772 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6773 WaitForEncodedFrame(timestamp_ms);
6774 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6775 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6776 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6777 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6778 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6779 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6780 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6781
6782 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6783 // expect increased resolution (960x540@30fps).
6784 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006785 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006786 timestamp_ms += kFrameIntervalMs;
6787 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006788 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006789 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006790 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6791 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6792 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6793 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6794 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006795 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006796
Evan Shrubsole64469032020-06-11 10:45:29 +02006797 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6798 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006799 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006800 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006801 timestamp_ms += kFrameIntervalMs;
6802 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006803 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006804 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006805 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006806 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6807 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6808 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6809 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6810 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006811 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006812
6813 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006814 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006815 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006816 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006817 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006818
mflodmancc3d4422017-08-03 08:27:51 -07006819 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006820}
6821
mflodmancc3d4422017-08-03 08:27:51 -07006822TEST_F(VideoStreamEncoderTest,
6823 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006824 const int kWidth = 640;
6825 const int kHeight = 360;
6826 const int kFpsLimit = 15;
6827 const int64_t kFrameIntervalMs = 150;
6828 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006829 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006830 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006831
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006832 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006833 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006834 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006835 video_stream_encoder_->SetSource(&source,
6836 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006837 timestamp_ms += kFrameIntervalMs;
6838 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006839 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006840 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006841 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6842 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6843 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6844 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6845 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6846 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6847
6848 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006849 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006850 timestamp_ms += kFrameIntervalMs;
6851 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006852 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006853 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006854 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6855 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6856 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6857 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6858 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6859 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6860
6861 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006862 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006863 timestamp_ms += kFrameIntervalMs;
6864 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006865 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006866 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006867 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006868 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006869 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6870 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6871 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6872 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6873
Evan Shrubsole64469032020-06-11 10:45:29 +02006874 // Trigger cpu adapt up, expect no change because quality is most limited.
6875 {
6876 auto previous_sink_wants = source.sink_wants();
6877 // Store current sink wants since we expect no change ind if there is no
6878 // change then last__wants() is not updated.
6879 video_stream_encoder_->TriggerCpuUnderuse();
6880 timestamp_ms += kFrameIntervalMs;
6881 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6882 WaitForEncodedFrame(timestamp_ms);
6883 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6884 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6885 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6886 }
6887
6888 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6889 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006890 timestamp_ms += kFrameIntervalMs;
6891 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006892 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006893 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006894 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6895 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6896 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006897 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6898 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6899 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006900
Evan Shrubsole64469032020-06-11 10:45:29 +02006901 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006902 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006903 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006904 timestamp_ms += kFrameIntervalMs;
6905 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006906 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006907 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006908 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6909 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6910 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6911 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6912 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006913 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006914
6915 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006916 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006917 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006918 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006919 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006920
mflodmancc3d4422017-08-03 08:27:51 -07006921 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006922}
6923
mflodmancc3d4422017-08-03 08:27:51 -07006924TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006925 const int kFrameWidth = 1920;
6926 const int kFrameHeight = 1080;
Ying Wangdb0d5862022-04-28 19:26:22 +02006927 // 2/3 of 1920.
6928 const int kAdaptedFrameWidth = 1280;
6929 // 2/3 of 1080.
6930 const int kAdaptedFrameHeight = 720;
ilnik6b826ef2017-06-16 06:53:48 -07006931 const int kFramerate = 24;
6932
Henrik Boström381d1092020-05-12 18:49:07 +02006933 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006934 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006935 // Trigger reconfigure encoder (without resetting the entire instance).
6936 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006937 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6938 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02006939 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 06:53:48 -07006940 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006941 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006942 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006943 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006944 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006945
6946 video_source_.set_adaptation_enabled(true);
6947
6948 video_source_.IncomingCapturedFrame(
6949 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006950 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006951
6952 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006953 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006954 video_source_.IncomingCapturedFrame(
6955 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006956 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006957
mflodmancc3d4422017-08-03 08:27:51 -07006958 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006959}
6960
mflodmancc3d4422017-08-03 08:27:51 -07006961TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006962 const int kFrameWidth = 1280;
6963 const int kFrameHeight = 720;
6964 const int kLowFps = 2;
6965 const int kHighFps = 30;
6966
Henrik Boström381d1092020-05-12 18:49:07 +02006967 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006968 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006969
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006970 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006971 max_framerate_ = kLowFps;
6972
6973 // Insert 2 seconds of 2fps video.
6974 for (int i = 0; i < kLowFps * 2; ++i) {
6975 video_source_.IncomingCapturedFrame(
6976 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6977 WaitForEncodedFrame(timestamp_ms);
6978 timestamp_ms += 1000 / kLowFps;
6979 }
6980
6981 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006982 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006983 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006984 video_source_.IncomingCapturedFrame(
6985 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6986 WaitForEncodedFrame(timestamp_ms);
6987 timestamp_ms += 1000 / kLowFps;
6988
6989 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6990
6991 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006992 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006993 const int kFrameIntervalMs = 1000 / kHighFps;
6994 max_framerate_ = kHighFps;
6995 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6996 video_source_.IncomingCapturedFrame(
6997 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6998 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6999 // be dropped if the encoder hans't been updated with the new higher target
7000 // framerate yet, causing it to overshoot the target bitrate and then
7001 // suffering the wrath of the media optimizer.
7002 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
7003 timestamp_ms += kFrameIntervalMs;
7004 }
7005
7006 // Don expect correct measurement just yet, but it should be higher than
7007 // before.
7008 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
7009
mflodmancc3d4422017-08-03 08:27:51 -07007010 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007011}
7012
mflodmancc3d4422017-08-03 08:27:51 -07007013TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07007014 const int kFrameWidth = 1280;
7015 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 15:09:05 +02007016 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01007017 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02007018 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07007019
Henrik Boström381d1092020-05-12 18:49:07 +02007020 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007021 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07007022 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07007023
7024 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007025 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07007026 video_source_.IncomingCapturedFrame(
7027 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7028 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02007029 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007030
7031 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02007032 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007033 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07007034
7035 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02007036 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007037 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07007038
Per Kjellanderdcef6412020-10-07 15:09:05 +02007039 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07007040 video_source_.IncomingCapturedFrame(
7041 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7042 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02007043 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007044
mflodmancc3d4422017-08-03 08:27:51 -07007045 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007046}
ilnik6b826ef2017-06-16 06:53:48 -07007047
Niels Möller4db138e2018-04-19 09:04:13 +02007048TEST_F(VideoStreamEncoderTest,
7049 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
7050 const int kFrameWidth = 1280;
7051 const int kFrameHeight = 720;
Markus Handell8e4197b2022-05-30 15:45:28 +02007052 const test::ScopedKeyValueConfig kFieldTrials;
7053 const CpuOveruseOptions default_options(kFieldTrials);
Henrik Boström381d1092020-05-12 18:49:07 +02007054 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007055 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007056 video_source_.IncomingCapturedFrame(
7057 CreateFrame(1, kFrameWidth, kFrameHeight));
7058 WaitForEncodedFrame(1);
7059 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7060 .low_encode_usage_threshold_percent,
7061 default_options.low_encode_usage_threshold_percent);
7062 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7063 .high_encode_usage_threshold_percent,
7064 default_options.high_encode_usage_threshold_percent);
7065 video_stream_encoder_->Stop();
7066}
7067
7068TEST_F(VideoStreamEncoderTest,
7069 HigherCpuAdaptationThresholdsForHardwareEncoder) {
7070 const int kFrameWidth = 1280;
7071 const int kFrameHeight = 720;
Markus Handell8e4197b2022-05-30 15:45:28 +02007072 const test::ScopedKeyValueConfig kFieldTrials;
7073 CpuOveruseOptions hardware_options(kFieldTrials);
Niels Möller4db138e2018-04-19 09:04:13 +02007074 hardware_options.low_encode_usage_threshold_percent = 150;
7075 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01007076 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02007077
Henrik Boström381d1092020-05-12 18:49:07 +02007078 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007079 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007080 video_source_.IncomingCapturedFrame(
7081 CreateFrame(1, kFrameWidth, kFrameHeight));
7082 WaitForEncodedFrame(1);
7083 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7084 .low_encode_usage_threshold_percent,
7085 hardware_options.low_encode_usage_threshold_percent);
7086 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7087 .high_encode_usage_threshold_percent,
7088 hardware_options.high_encode_usage_threshold_percent);
7089 video_stream_encoder_->Stop();
7090}
7091
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007092TEST_F(VideoStreamEncoderTest,
7093 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
7094 const int kFrameWidth = 1280;
7095 const int kFrameHeight = 720;
7096
Markus Handell8e4197b2022-05-30 15:45:28 +02007097 const test::ScopedKeyValueConfig kFieldTrials;
7098 const CpuOveruseOptions default_options(kFieldTrials);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007099 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007100 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007101 video_source_.IncomingCapturedFrame(
7102 CreateFrame(1, kFrameWidth, kFrameHeight));
7103 WaitForEncodedFrame(1);
7104 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7105 .low_encode_usage_threshold_percent,
7106 default_options.low_encode_usage_threshold_percent);
7107 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7108 .high_encode_usage_threshold_percent,
7109 default_options.high_encode_usage_threshold_percent);
7110
Markus Handell8e4197b2022-05-30 15:45:28 +02007111 CpuOveruseOptions hardware_options(kFieldTrials);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007112 hardware_options.low_encode_usage_threshold_percent = 150;
7113 hardware_options.high_encode_usage_threshold_percent = 200;
7114 fake_encoder_.SetIsHardwareAccelerated(true);
7115
7116 video_source_.IncomingCapturedFrame(
7117 CreateFrame(2, kFrameWidth, kFrameHeight));
7118 WaitForEncodedFrame(2);
7119
7120 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7121 .low_encode_usage_threshold_percent,
7122 hardware_options.low_encode_usage_threshold_percent);
7123 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7124 .high_encode_usage_threshold_percent,
7125 hardware_options.high_encode_usage_threshold_percent);
7126
7127 video_stream_encoder_->Stop();
7128}
7129
Niels Möller6bb5ab92019-01-11 11:11:10 +01007130TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7131 const int kFrameWidth = 320;
7132 const int kFrameHeight = 240;
7133 const int kFps = 30;
Asa Persson606d3cb2021-10-04 10:07:11 +02007134 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007135 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7136
Henrik Boström381d1092020-05-12 18:49:07 +02007137 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007138 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007139
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007140 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007141 max_framerate_ = kFps;
7142
7143 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7144 fake_encoder_.SimulateOvershoot(1.0);
7145 int num_dropped = 0;
7146 for (int i = 0; i < kNumFramesInRun; ++i) {
7147 video_source_.IncomingCapturedFrame(
7148 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7149 // Wait up to two frame durations for a frame to arrive.
7150 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7151 ++num_dropped;
7152 }
7153 timestamp_ms += 1000 / kFps;
7154 }
7155
Erik Språnga8d48ab2019-02-08 14:17:40 +01007156 // Framerate should be measured to be near the expected target rate.
7157 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7158
7159 // Frame drops should be within 5% of expected 0%.
7160 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007161
7162 // Make encoder produce frames at double the expected bitrate during 3 seconds
7163 // of video, verify number of drops. Rate needs to be slightly changed in
7164 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01007165 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02007166 const RateControlSettings trials =
7167 RateControlSettings::ParseFromFieldTrials();
7168 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01007169 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02007170 // frame dropping since the adjuter will try to just lower the target
7171 // bitrate rather than drop frames. If network headroom can be used, it
7172 // doesn't push back as hard so we don't need quite as much overshoot.
7173 // These numbers are unfortunately a bit magical but there's not trivial
7174 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01007175 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01007176 }
7177 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02007178 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007179 kTargetBitrate + DataRate::KilobitsPerSec(1),
7180 kTargetBitrate + DataRate::KilobitsPerSec(1),
7181 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007182 num_dropped = 0;
7183 for (int i = 0; i < kNumFramesInRun; ++i) {
7184 video_source_.IncomingCapturedFrame(
7185 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7186 // Wait up to two frame durations for a frame to arrive.
7187 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7188 ++num_dropped;
7189 }
7190 timestamp_ms += 1000 / kFps;
7191 }
7192
Henrik Boström381d1092020-05-12 18:49:07 +02007193 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007194 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007195
7196 // Target framerate should be still be near the expected target, despite
7197 // the frame drops.
7198 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7199
7200 // Frame drops should be within 5% of expected 50%.
7201 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007202
7203 video_stream_encoder_->Stop();
7204}
7205
7206TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7207 const int kFrameWidth = 320;
7208 const int kFrameHeight = 240;
7209 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 10:07:11 +02007210 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007211
7212 ASSERT_GT(max_framerate_, kActualInputFps);
7213
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007214 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007215 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007216 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007217 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007218
7219 // Insert 3 seconds of video, with an input fps lower than configured max.
7220 for (int i = 0; i < kActualInputFps * 3; ++i) {
7221 video_source_.IncomingCapturedFrame(
7222 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7223 // Wait up to two frame durations for a frame to arrive.
7224 WaitForEncodedFrame(timestamp_ms);
7225 timestamp_ms += 1000 / kActualInputFps;
7226 }
7227
7228 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7229
7230 video_stream_encoder_->Stop();
7231}
7232
Markus Handell9a478b52021-11-18 16:07:01 +01007233TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007234 VideoFrame::UpdateRect rect;
Markus Handell9a478b52021-11-18 16:07:01 +01007235 test::FrameForwarder source;
7236 video_stream_encoder_->SetSource(&source,
7237 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02007238 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007239 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007240
Markus Handell9a478b52021-11-18 16:07:01 +01007241 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(1, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007242 WaitForEncodedFrame(1);
7243 // On the very first frame full update should be forced.
7244 rect = fake_encoder_.GetLastUpdateRect();
7245 EXPECT_EQ(rect.offset_x, 0);
7246 EXPECT_EQ(rect.offset_y, 0);
7247 EXPECT_EQ(rect.height, codec_height_);
7248 EXPECT_EQ(rect.width, codec_width_);
Markus Handell9a478b52021-11-18 16:07:01 +01007249 // Frame with NTP timestamp 2 will be dropped due to outstanding frames
7250 // scheduled for processing during encoder queue processing of frame 2.
7251 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(2, nullptr, 1));
7252 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(3, nullptr, 10));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007253 WaitForEncodedFrame(3);
7254 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7255 rect = fake_encoder_.GetLastUpdateRect();
7256 EXPECT_EQ(rect.offset_x, 1);
7257 EXPECT_EQ(rect.offset_y, 0);
7258 EXPECT_EQ(rect.width, 10);
7259 EXPECT_EQ(rect.height, 1);
7260
Markus Handell9a478b52021-11-18 16:07:01 +01007261 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(4, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007262 WaitForEncodedFrame(4);
7263 // Previous frame was encoded, so no accumulation should happen.
7264 rect = fake_encoder_.GetLastUpdateRect();
7265 EXPECT_EQ(rect.offset_x, 0);
7266 EXPECT_EQ(rect.offset_y, 0);
7267 EXPECT_EQ(rect.width, 1);
7268 EXPECT_EQ(rect.height, 1);
7269
7270 video_stream_encoder_->Stop();
7271}
7272
Erik Språngd7329ca2019-02-21 21:19:53 +01007273TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007274 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007275 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007276
7277 // First frame is always keyframe.
7278 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7279 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007280 EXPECT_THAT(
7281 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007282 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007283
7284 // Insert delta frame.
7285 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7286 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007287 EXPECT_THAT(
7288 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007289 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007290
7291 // Request next frame be a key-frame.
7292 video_stream_encoder_->SendKeyFrame();
7293 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7294 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007295 EXPECT_THAT(
7296 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007297 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007298
7299 video_stream_encoder_->Stop();
7300}
7301
7302TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7303 // Setup simulcast with three streams.
7304 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007305 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007306 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7307 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007308 // Wait for all three layers before triggering event.
7309 sink_.SetNumExpectedLayers(3);
7310
7311 // First frame is always keyframe.
7312 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7313 WaitForEncodedFrame(1);
7314 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007315 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7316 VideoFrameType::kVideoFrameKey,
7317 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007318
7319 // Insert delta frame.
7320 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7321 WaitForEncodedFrame(2);
7322 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007323 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7324 VideoFrameType::kVideoFrameDelta,
7325 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007326
7327 // Request next frame be a key-frame.
7328 // Only first stream is configured to produce key-frame.
7329 video_stream_encoder_->SendKeyFrame();
7330 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7331 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007332
7333 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7334 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007335 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007336 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007337 VideoFrameType::kVideoFrameKey,
7338 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007339
7340 video_stream_encoder_->Stop();
7341}
7342
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007343TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007344 // SPS contains VUI with restrictions on the maximum number of reordered
7345 // pictures, there is no need to rewrite the bitstream to enable faster
7346 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007347 ResetEncoder("H264", 1, 1, 1, false);
7348
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007349 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007350 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007351 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007352
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007353 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 10:07:11 +02007354 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007355
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007356 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7357 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007358
7359 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007360 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007361
7362 video_stream_encoder_->Stop();
7363}
7364
7365TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007366 // SPS does not contain VUI, the bitstream is will be rewritten with added
7367 // VUI with restrictions on the maximum number of reordered pictures to
7368 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007369 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7370 0x00, 0x00, 0x03, 0x03, 0xF4,
7371 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007372 ResetEncoder("H264", 1, 1, 1, false);
7373
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007374 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007375 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007376 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007377
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007378 fake_encoder_.SetEncodedImageData(
7379 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007380
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007381 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7382 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007383
7384 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007385 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007386
7387 video_stream_encoder_->Stop();
7388}
7389
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007390TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7391 const int kFrameWidth = 1280;
7392 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 10:07:11 +02007393 const DataRate kTargetBitrate =
7394 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007395
Henrik Boström381d1092020-05-12 18:49:07 +02007396 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007397 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007398 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7399
7400 // Insert a first video frame. It should be dropped because of downscale in
7401 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007402 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007403 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7404 frame.set_rotation(kVideoRotation_270);
7405 video_source_.IncomingCapturedFrame(frame);
7406
7407 ExpectDroppedFrame();
7408
7409 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007410 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007411 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7412 frame.set_rotation(kVideoRotation_90);
7413 video_source_.IncomingCapturedFrame(frame);
7414
7415 WaitForEncodedFrame(timestamp_ms);
7416 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7417
7418 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007419 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007420 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7421 frame.set_rotation(kVideoRotation_180);
7422 video_source_.IncomingCapturedFrame(frame);
7423
7424 WaitForEncodedFrame(timestamp_ms);
7425 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7426
7427 video_stream_encoder_->Stop();
7428}
7429
Erik Språng5056af02019-09-02 15:53:11 +02007430TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7431 const int kFrameWidth = 320;
7432 const int kFrameHeight = 180;
7433
7434 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007435 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007436 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7437 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7438 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007439 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007440 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007441 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007442
7443 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007444 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007445 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7446 frame.set_rotation(kVideoRotation_270);
7447 video_source_.IncomingCapturedFrame(frame);
7448 WaitForEncodedFrame(timestamp_ms);
7449
7450 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 10:07:11 +02007451 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007452 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7453 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007454 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007455 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007456 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007457 /*link_allocation=*/target_rate,
7458 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007459 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007460 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007461 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7462
7463 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7464 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7465 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007466 DataRate allocation_sum =
7467 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007468 EXPECT_EQ(min_rate, allocation_sum);
7469 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7470
7471 video_stream_encoder_->Stop();
7472}
7473
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007474TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007475 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007476 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007477 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007478 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007479 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7480 WaitForEncodedFrame(1);
7481
7482 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7483 ASSERT_TRUE(prev_rate_settings.has_value());
7484 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7485 kDefaultFramerate);
7486
7487 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7488 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7489 timestamp_ms += 1000 / kDefaultFramerate;
7490 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7491 WaitForEncodedFrame(timestamp_ms);
7492 }
7493 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7494 kDefaultFramerate);
7495 // Capture larger frame to trigger a reconfigure.
7496 codec_height_ *= 2;
7497 codec_width_ *= 2;
7498 timestamp_ms += 1000 / kDefaultFramerate;
7499 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7500 WaitForEncodedFrame(timestamp_ms);
7501
7502 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7503 auto current_rate_settings =
7504 fake_encoder_.GetAndResetLastRateControlSettings();
7505 // Ensure we have actually reconfigured twice
7506 // The rate settings should have been set again even though
7507 // they haven't changed.
7508 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007509 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007510
7511 video_stream_encoder_->Stop();
7512}
7513
philipeld9cc8c02019-09-16 14:53:40 +02007514struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007515 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007516 MOCK_METHOD(void,
7517 RequestEncoderSwitch,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007518 (const webrtc::SdpVideoFormat& format,
7519 bool allow_default_fallback),
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007520 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007521};
7522
philipel9b058032020-02-10 11:30:00 +01007523TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7524 constexpr int kDontCare = 100;
7525 StrictMock<MockEncoderSelector> encoder_selector;
7526 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7527 &fake_encoder_, &encoder_selector);
7528 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7529
7530 // Reset encoder for new configuration to take effect.
7531 ConfigureEncoder(video_encoder_config_.Copy());
7532
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007533 EXPECT_CALL(encoder_selector, OnCurrentEncoder);
philipel9b058032020-02-10 11:30:00 +01007534
7535 video_source_.IncomingCapturedFrame(
7536 CreateFrame(kDontCare, kDontCare, kDontCare));
Markus Handell28c71802021-11-08 10:11:55 +01007537 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007538 video_stream_encoder_->Stop();
7539
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007540 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007541 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007542 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7543 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007544 video_stream_encoder_.reset();
7545}
7546
7547TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7548 constexpr int kDontCare = 100;
7549
7550 NiceMock<MockEncoderSelector> encoder_selector;
7551 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7552 video_send_config_.encoder_settings.encoder_switch_request_callback =
7553 &switch_callback;
7554 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7555 &fake_encoder_, &encoder_selector);
7556 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7557
7558 // Reset encoder for new configuration to take effect.
7559 ConfigureEncoder(video_encoder_config_.Copy());
7560
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007561 ON_CALL(encoder_selector, OnAvailableBitrate)
philipel9b058032020-02-10 11:30:00 +01007562 .WillByDefault(Return(SdpVideoFormat("AV1")));
7563 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007564 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7565 /*allow_default_fallback=*/false));
philipel9b058032020-02-10 11:30:00 +01007566
Henrik Boström381d1092020-05-12 18:49:07 +02007567 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007568 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7569 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7570 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007571 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007572 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007573 /*cwnd_reduce_ratio=*/0);
Markus Handell28c71802021-11-08 10:11:55 +01007574 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007575
7576 video_stream_encoder_->Stop();
7577}
7578
philipel6daa3042022-04-11 10:48:28 +02007579TEST_F(VideoStreamEncoderTest, EncoderSelectorResolutionSwitch) {
7580 NiceMock<MockEncoderSelector> encoder_selector;
7581 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7582 video_send_config_.encoder_settings.encoder_switch_request_callback =
7583 &switch_callback;
7584 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7585 &fake_encoder_, &encoder_selector);
7586 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7587
7588 // Reset encoder for new configuration to take effect.
7589 ConfigureEncoder(video_encoder_config_.Copy());
7590
7591 EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(640, 480)))
7592 .WillOnce(Return(absl::nullopt));
7593 EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(320, 240)))
7594 .WillOnce(Return(SdpVideoFormat("AV1")));
7595 EXPECT_CALL(switch_callback,
7596 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7597 /*allow_default_fallback=*/false));
7598
7599 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7600 /*target_bitrate=*/DataRate::KilobitsPerSec(800),
7601 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(1000),
7602 /*link_allocation=*/DataRate::KilobitsPerSec(1000),
7603 /*fraction_lost=*/0,
7604 /*round_trip_time_ms=*/0,
7605 /*cwnd_reduce_ratio=*/0);
7606
7607 video_source_.IncomingCapturedFrame(CreateFrame(1, 640, 480));
7608 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 480));
7609 video_source_.IncomingCapturedFrame(CreateFrame(3, 320, 240));
7610
7611 AdvanceTime(TimeDelta::Zero());
7612
7613 video_stream_encoder_->Stop();
7614}
7615
philipel9b058032020-02-10 11:30:00 +01007616TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7617 constexpr int kSufficientBitrateToNotDrop = 1000;
7618 constexpr int kDontCare = 100;
7619
7620 NiceMock<MockVideoEncoder> video_encoder;
7621 NiceMock<MockEncoderSelector> encoder_selector;
7622 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7623 video_send_config_.encoder_settings.encoder_switch_request_callback =
7624 &switch_callback;
7625 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7626 &video_encoder, &encoder_selector);
7627 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7628
7629 // Reset encoder for new configuration to take effect.
7630 ConfigureEncoder(video_encoder_config_.Copy());
7631
7632 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7633 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7634 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007635 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007636 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7637 /*stable_target_bitrate=*/
7638 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7639 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007640 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007641 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007642 /*cwnd_reduce_ratio=*/0);
7643
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007644 ON_CALL(video_encoder, Encode)
philipel9b058032020-02-10 11:30:00 +01007645 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007646 ON_CALL(encoder_selector, OnEncoderBroken)
philipel9b058032020-02-10 11:30:00 +01007647 .WillByDefault(Return(SdpVideoFormat("AV2")));
7648
7649 rtc::Event encode_attempted;
7650 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007651 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7652 /*allow_default_fallback=*/true))
7653 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
philipel9b058032020-02-10 11:30:00 +01007654
7655 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7656 encode_attempted.Wait(3000);
7657
Markus Handell28c71802021-11-08 10:11:55 +01007658 AdvanceTime(TimeDelta::Zero());
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007659
philipel9b058032020-02-10 11:30:00 +01007660 video_stream_encoder_->Stop();
7661
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007662 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7663 // to it's factory, so in order for the encoder instance in the
7664 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7665 // reset the `video_stream_encoder_` here.
7666 video_stream_encoder_.reset();
7667}
7668
7669TEST_F(VideoStreamEncoderTest, SwitchEncoderOnInitFailureWithEncoderSelector) {
7670 NiceMock<MockVideoEncoder> video_encoder;
7671 NiceMock<MockEncoderSelector> encoder_selector;
7672 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7673 video_send_config_.encoder_settings.encoder_switch_request_callback =
7674 &switch_callback;
7675 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7676 &video_encoder, &encoder_selector);
7677 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7678
7679 // Reset encoder for new configuration to take effect.
7680 ConfigureEncoder(video_encoder_config_.Copy());
7681
7682 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7683 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7684 /*round_trip_time_ms=*/0,
7685 /*cwnd_reduce_ratio=*/0);
7686 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7687
7688 ON_CALL(video_encoder, InitEncode(_, _))
7689 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7690 ON_CALL(encoder_selector, OnEncoderBroken)
7691 .WillByDefault(Return(SdpVideoFormat("AV2")));
7692
7693 rtc::Event encode_attempted;
7694 EXPECT_CALL(switch_callback,
7695 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7696 /*allow_default_fallback=*/true))
7697 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7698
7699 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7700 encode_attempted.Wait(3000);
7701
7702 AdvanceTime(TimeDelta::Zero());
7703
7704 video_stream_encoder_->Stop();
7705
7706 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7707 // to it's factory, so in order for the encoder instance in the
7708 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7709 // reset the `video_stream_encoder_` here.
7710 video_stream_encoder_.reset();
7711}
7712
7713TEST_F(VideoStreamEncoderTest,
7714 SwitchEncoderOnInitFailureWithoutEncoderSelector) {
7715 NiceMock<MockVideoEncoder> video_encoder;
7716 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7717 video_send_config_.encoder_settings.encoder_switch_request_callback =
7718 &switch_callback;
7719 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7720 &video_encoder, /*encoder_selector=*/nullptr);
7721 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7722
7723 // Reset encoder for new configuration to take effect.
7724 ConfigureEncoder(video_encoder_config_.Copy());
7725
7726 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7727 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7728 /*round_trip_time_ms=*/0,
7729 /*cwnd_reduce_ratio=*/0);
7730 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7731
7732 ON_CALL(video_encoder, InitEncode(_, _))
7733 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7734
7735 rtc::Event encode_attempted;
7736 EXPECT_CALL(switch_callback,
7737 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "VP8"),
7738 /*allow_default_fallback=*/true))
7739 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7740
7741 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7742 encode_attempted.Wait(3000);
7743
7744 AdvanceTime(TimeDelta::Zero());
7745
7746 video_stream_encoder_->Stop();
7747
7748 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007749 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007750 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7751 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007752 video_stream_encoder_.reset();
7753}
7754
Byoungchan Lee13fe3672022-04-06 10:44:42 +09007755TEST_F(VideoStreamEncoderTest, NullEncoderReturnSwitch) {
7756 // As a variant of EncoderSelectorBrokenEncoderSwitch, when a null
7757 // VideoEncoder is passed in encoder_factory, it checks whether
7758 // Codec Switch occurs without a crash.
7759 constexpr int kSufficientBitrateToNotDrop = 1000;
7760 constexpr int kDontCare = 100;
7761
7762 NiceMock<MockEncoderSelector> encoder_selector;
7763 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7764 video_send_config_.encoder_settings.encoder_switch_request_callback =
7765 &switch_callback;
7766 auto encoder_factory =
7767 std::make_unique<test::VideoEncoderNullableProxyFactory>(
7768 /*encoder=*/nullptr, &encoder_selector);
7769 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7770
7771 // Reset encoder for new configuration to take effect.
7772 ConfigureEncoder(video_encoder_config_.Copy());
7773 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7774 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7775 // not fail.
7776 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7777 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7778 /*stable_target_bitrate=*/
7779 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7780 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7781 /*fraction_lost=*/0,
7782 /*round_trip_time_ms=*/0,
7783 /*cwnd_reduce_ratio=*/0);
7784 ON_CALL(encoder_selector, OnEncoderBroken)
7785 .WillByDefault(Return(SdpVideoFormat("AV2")));
7786 rtc::Event encode_attempted;
7787 EXPECT_CALL(switch_callback,
7788 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7789 /*allow_default_fallback=*/_))
7790 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7791
7792 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7793 encode_attempted.Wait(3000);
7794
7795 AdvanceTime(TimeDelta::Zero());
7796
7797 video_stream_encoder_->Stop();
7798
7799 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7800 // to it's factory, so in order for the encoder instance in the
7801 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7802 // reset the `video_stream_encoder_` here.
7803 video_stream_encoder_.reset();
7804}
7805
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007806TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007807 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007808 const int kFrameWidth = 320;
7809 const int kFrameHeight = 180;
7810
7811 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007812 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007813 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007814 /*target_bitrate=*/rate,
7815 /*stable_target_bitrate=*/rate,
7816 /*link_allocation=*/rate,
7817 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007818 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007819 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007820
7821 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007822 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007823 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7824 frame.set_rotation(kVideoRotation_270);
7825 video_source_.IncomingCapturedFrame(frame);
7826 WaitForEncodedFrame(timestamp_ms);
7827 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7828
7829 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007830 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007831 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007832 /*target_bitrate=*/new_stable_rate,
7833 /*stable_target_bitrate=*/new_stable_rate,
7834 /*link_allocation=*/rate,
7835 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007836 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007837 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007838 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7839 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7840 video_stream_encoder_->Stop();
7841}
7842
7843TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007844 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007845 const int kFrameWidth = 320;
7846 const int kFrameHeight = 180;
7847
7848 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007849 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007850 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007851 /*target_bitrate=*/rate,
7852 /*stable_target_bitrate=*/rate,
7853 /*link_allocation=*/rate,
7854 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007855 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007856 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007857
7858 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007859 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007860 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7861 frame.set_rotation(kVideoRotation_270);
7862 video_source_.IncomingCapturedFrame(frame);
7863 WaitForEncodedFrame(timestamp_ms);
7864 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7865
7866 // Set a higher target rate without changing the link_allocation. Should not
7867 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007868 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007869 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007870 /*target_bitrate=*/rate,
7871 /*stable_target_bitrate=*/new_stable_rate,
7872 /*link_allocation=*/rate,
7873 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007874 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007875 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007876 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7877 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7878 video_stream_encoder_->Stop();
7879}
7880
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007881TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01007882 test::ScopedKeyValueConfig field_trials(
7883 field_trials_,
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007884 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7885 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7886 const int kFramerateFps = 30;
7887 const int kWidth = 1920;
7888 const int kHeight = 1080;
7889 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7890 // Works on screenshare mode.
7891 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7892 // We rely on the automatic resolution adaptation, but we handle framerate
7893 // adaptation manually by mocking the stats proxy.
7894 video_source_.set_adaptation_enabled(true);
7895
7896 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007897 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007898 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007899 video_stream_encoder_->SetSource(&video_source_,
7900 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007901 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007902
7903 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7904 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7905
7906 // Pass enough frames with the full update to trigger animation detection.
7907 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007908 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007909 frame.set_ntp_time_ms(timestamp_ms);
7910 frame.set_timestamp_us(timestamp_ms * 1000);
7911 video_source_.IncomingCapturedFrame(frame);
7912 WaitForEncodedFrame(timestamp_ms);
7913 }
7914
7915 // Resolution should be limited.
7916 rtc::VideoSinkWants expected;
7917 expected.max_framerate_fps = kFramerateFps;
7918 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007919 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007920
7921 // Pass one frame with no known update.
7922 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007923 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007924 frame.set_ntp_time_ms(timestamp_ms);
7925 frame.set_timestamp_us(timestamp_ms * 1000);
7926 frame.clear_update_rect();
7927
7928 video_source_.IncomingCapturedFrame(frame);
7929 WaitForEncodedFrame(timestamp_ms);
7930
7931 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007932 EXPECT_THAT(video_source_.sink_wants(),
7933 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007934
7935 video_stream_encoder_->Stop();
7936}
7937
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007938TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7939 const int kWidth = 720; // 540p adapted down.
7940 const int kHeight = 405;
7941 const int kNumFrames = 3;
7942 // Works on screenshare mode.
7943 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7944 /*num_spatial_layers=*/2, /*screenshare=*/true);
7945
7946 video_source_.set_adaptation_enabled(true);
7947
7948 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007949 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007950
7951 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7952
7953 // Pass enough frames with the full update to trigger animation detection.
7954 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007955 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007956 frame.set_ntp_time_ms(timestamp_ms);
7957 frame.set_timestamp_us(timestamp_ms * 1000);
7958 video_source_.IncomingCapturedFrame(frame);
7959 WaitForEncodedFrame(timestamp_ms);
7960 }
7961
7962 video_stream_encoder_->Stop();
7963}
7964
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007965TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7966 const float downscale_factors[] = {4.0, 2.0, 1.0};
7967 const int number_layers =
7968 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7969 VideoEncoderConfig config;
7970 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7971 for (int i = 0; i < number_layers; ++i) {
7972 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7973 config.simulcast_layers[i].active = true;
7974 }
7975 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007976 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007977 "VP8", /*max qp*/ 56, /*screencast*/ false,
7978 /*screenshare enabled*/ false);
7979 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007980 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7981 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007982
7983 // First initialization.
7984 // Encoder should be initialized. Next frame should be key frame.
7985 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7986 sink_.SetNumExpectedLayers(number_layers);
7987 int64_t timestamp_ms = kFrameIntervalMs;
7988 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7989 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007990 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007991 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7992 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7993 VideoFrameType::kVideoFrameKey,
7994 VideoFrameType::kVideoFrameKey}));
7995
7996 // Disable top layer.
7997 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7998 config.simulcast_layers[number_layers - 1].active = false;
7999 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8000 sink_.SetNumExpectedLayers(number_layers - 1);
8001 timestamp_ms += kFrameIntervalMs;
8002 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8003 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008004 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008005 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8006 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
8007 VideoFrameType::kVideoFrameDelta,
8008 VideoFrameType::kVideoFrameDelta}));
8009
8010 // Re-enable top layer.
8011 // Encoder should be re-initialized. Next frame should be key frame.
8012 config.simulcast_layers[number_layers - 1].active = true;
8013 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8014 sink_.SetNumExpectedLayers(number_layers);
8015 timestamp_ms += kFrameIntervalMs;
8016 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8017 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008018 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008019 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8020 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
8021 VideoFrameType::kVideoFrameKey,
8022 VideoFrameType::kVideoFrameKey}));
8023
8024 // Top layer max rate change.
8025 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
8026 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
8027 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8028 sink_.SetNumExpectedLayers(number_layers);
8029 timestamp_ms += kFrameIntervalMs;
8030 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8031 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008032 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008033 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8034 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
8035 VideoFrameType::kVideoFrameDelta,
8036 VideoFrameType::kVideoFrameDelta}));
8037
8038 // Top layer resolution change.
8039 // Encoder should be re-initialized. Next frame should be key frame.
8040 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
8041 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8042 sink_.SetNumExpectedLayers(number_layers);
8043 timestamp_ms += kFrameIntervalMs;
8044 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8045 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008046 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008047 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8048 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
8049 VideoFrameType::kVideoFrameKey,
8050 VideoFrameType::kVideoFrameKey}));
8051 video_stream_encoder_->Stop();
8052}
8053
Henrik Boström1124ed12021-02-25 10:30:39 +01008054TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
8055 const int kFrameWidth = 1280;
8056 const int kFrameHeight = 720;
8057
8058 SetUp();
8059 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008060 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01008061
8062 // Capturing a frame should reconfigure the encoder and expose the encoder
8063 // resolution, which is the same as the input frame.
8064 int64_t timestamp_ms = kFrameIntervalMs;
8065 video_source_.IncomingCapturedFrame(
8066 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8067 WaitForEncodedFrame(timestamp_ms);
8068 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8069 EXPECT_THAT(video_source_.sink_wants().resolutions,
8070 ::testing::ElementsAreArray(
8071 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
8072
8073 video_stream_encoder_->Stop();
8074}
8075
8076TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
8077 // Pick downscale factors such that we never encode at full resolution - this
8078 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 16:22:11 +02008079 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 10:30:39 +01008080 // encoder should not ask for the frame resolution. This allows video frames
8081 // to have the appearence of one resolution but optimize its internal buffers
8082 // for what is actually encoded.
8083 const size_t kNumSimulcastLayers = 3u;
8084 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
8085 const int kFrameWidth = 1280;
8086 const int kFrameHeight = 720;
8087 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8088 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8089 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8090 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8091 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8092 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8093
8094 VideoEncoderConfig config;
8095 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
8096 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
8097 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
8098 config.simulcast_layers[i].active = true;
8099 }
8100 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02008101 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01008102 "VP8", /*max qp*/ 56, /*screencast*/ false,
8103 /*screenshare enabled*/ false);
8104 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008105 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8106 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01008107
8108 // Capture a frame with all layers active.
8109 int64_t timestamp_ms = kFrameIntervalMs;
8110 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
8111 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8112 video_source_.IncomingCapturedFrame(
8113 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8114 WaitForEncodedFrame(timestamp_ms);
8115 // Expect encoded resolutions to match the expected simulcast layers.
8116 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8117 EXPECT_THAT(
8118 video_source_.sink_wants().resolutions,
8119 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
8120
8121 // Capture a frame with one of the layers inactive.
8122 timestamp_ms += kFrameIntervalMs;
8123 config.simulcast_layers[2].active = false;
8124 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
8125 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8126 video_source_.IncomingCapturedFrame(
8127 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8128 WaitForEncodedFrame(timestamp_ms);
8129
8130 // Expect encoded resolutions to match the expected simulcast layers.
8131 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8132 EXPECT_THAT(video_source_.sink_wants().resolutions,
8133 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
8134
8135 // Capture a frame with all but one layer turned off.
8136 timestamp_ms += kFrameIntervalMs;
8137 config.simulcast_layers[1].active = false;
8138 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
8139 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8140 video_source_.IncomingCapturedFrame(
8141 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8142 WaitForEncodedFrame(timestamp_ms);
8143
8144 // Expect encoded resolutions to match the expected simulcast layers.
8145 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8146 EXPECT_THAT(video_source_.sink_wants().resolutions,
8147 ::testing::ElementsAreArray({kLayer0Size}));
8148
8149 video_stream_encoder_->Stop();
8150}
8151
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008152TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008153 ResetEncoder("VP8", 1, 1, 1, false);
8154
Niels Möller8b692902021-06-14 12:04:57 +02008155 // Force encoder reconfig.
8156 video_source_.IncomingCapturedFrame(
8157 CreateFrame(1, codec_width_, codec_height_));
8158 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8159
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008160 // Set QP on encoded frame and pass the frame to encode complete callback.
8161 // Since QP is present QP parsing won't be triggered and the original value
8162 // should be kept.
8163 EncodedImage encoded_image;
8164 encoded_image.qp_ = 123;
8165 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8166 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8167 CodecSpecificInfo codec_info;
8168 codec_info.codecType = kVideoCodecVP8;
8169 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8170 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8171 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
8172 video_stream_encoder_->Stop();
8173}
8174
8175TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008176 ResetEncoder("VP8", 1, 1, 1, false);
8177
Niels Möller8b692902021-06-14 12:04:57 +02008178 // Force encoder reconfig.
8179 video_source_.IncomingCapturedFrame(
8180 CreateFrame(1, codec_width_, codec_height_));
8181 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8182
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008183 // Pass an encoded frame without QP to encode complete callback. QP should be
8184 // parsed and set.
8185 EncodedImage encoded_image;
8186 encoded_image.qp_ = -1;
8187 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8188 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8189 CodecSpecificInfo codec_info;
8190 codec_info.codecType = kVideoCodecVP8;
8191 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8192 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8193 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
8194 video_stream_encoder_->Stop();
8195}
8196
8197TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01008198 webrtc::test::ScopedKeyValueConfig field_trials(
8199 field_trials_, "WebRTC-QpParsingKillSwitch/Enabled/");
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008200
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008201 ResetEncoder("VP8", 1, 1, 1, false);
8202
Niels Möller8b692902021-06-14 12:04:57 +02008203 // Force encoder reconfig.
8204 video_source_.IncomingCapturedFrame(
8205 CreateFrame(1, codec_width_, codec_height_));
8206 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8207
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008208 EncodedImage encoded_image;
8209 encoded_image.qp_ = -1;
8210 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8211 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8212 CodecSpecificInfo codec_info;
8213 codec_info.codecType = kVideoCodecVP8;
8214 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8215 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8216 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
8217 video_stream_encoder_->Stop();
8218}
8219
Sergey Silkind19e3b92021-03-16 10:05:30 +00008220TEST_F(VideoStreamEncoderTest,
8221 QualityScalingNotAllowed_QualityScalingDisabled) {
8222 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8223
8224 // Disable scaling settings in encoder info.
8225 fake_encoder_.SetQualityScaling(false);
8226 // Disable quality scaling in encoder config.
8227 video_encoder_config.is_quality_scaling_allowed = false;
8228 ConfigureEncoder(std::move(video_encoder_config));
8229
8230 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008231 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008232
8233 test::FrameForwarder source;
8234 video_stream_encoder_->SetSource(
8235 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8236 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8237 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8238
8239 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8240 WaitForEncodedFrame(1);
8241 video_stream_encoder_->TriggerQualityLow();
8242 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8243
8244 video_stream_encoder_->Stop();
8245}
8246
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008247TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
8248 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8249
8250 // Disable scaling settings in encoder info.
8251 fake_encoder_.SetQualityScaling(false);
8252 // Set QP trusted in encoder info.
8253 fake_encoder_.SetIsQpTrusted(true);
8254 // Enable quality scaling in encoder config.
8255 video_encoder_config.is_quality_scaling_allowed = false;
8256 ConfigureEncoder(std::move(video_encoder_config));
8257
8258 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008259 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008260
8261 test::FrameForwarder source;
8262 video_stream_encoder_->SetSource(
8263 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8264 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8265 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8266
8267 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8268 WaitForEncodedFrame(1);
8269 video_stream_encoder_->TriggerQualityLow();
8270 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8271
8272 video_stream_encoder_->Stop();
8273}
8274
Shuhai Pengf2707702021-09-29 17:19:44 +08008275TEST_F(VideoStreamEncoderTest,
8276 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8277 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8278
8279 // Disable scaling settings in encoder info.
8280 fake_encoder_.SetQualityScaling(false);
8281 // Set QP trusted in encoder info.
8282 fake_encoder_.SetIsQpTrusted(true);
8283 // Enable quality scaling in encoder config.
8284 video_encoder_config.is_quality_scaling_allowed = false;
8285 ConfigureEncoder(std::move(video_encoder_config));
8286
8287 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008288 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008289
8290 test::FrameForwarder source;
8291 video_stream_encoder_->SetSource(
8292 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8293 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8294 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8295
8296 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8297 WaitForEncodedFrame(1);
8298 video_stream_encoder_->TriggerQualityLow();
8299 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8300
8301 video_stream_encoder_->Stop();
8302}
8303
8304TEST_F(VideoStreamEncoderTest,
8305 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8306 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8307
8308 // Disable scaling settings in encoder info.
8309 fake_encoder_.SetQualityScaling(false);
8310 // Set QP trusted in encoder info.
8311 fake_encoder_.SetIsQpTrusted(false);
8312 // Enable quality scaling in encoder config.
8313 video_encoder_config.is_quality_scaling_allowed = false;
8314 ConfigureEncoder(std::move(video_encoder_config));
8315
8316 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008317 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008318
8319 test::FrameForwarder source;
8320 video_stream_encoder_->SetSource(
8321 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8322 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8323 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8324
8325 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8326 WaitForEncodedFrame(1);
8327 video_stream_encoder_->TriggerQualityLow();
8328 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8329
8330 video_stream_encoder_->Stop();
8331}
8332
8333TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8334 // Set QP trusted in encoder info.
8335 fake_encoder_.SetIsQpTrusted(false);
8336
8337 const int MinEncBitrateKbps = 30;
8338 const int MaxEncBitrateKbps = 100;
8339 const int MinStartBitrateKbp = 50;
8340 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8341 /*frame_size_pixels=*/codec_width_ * codec_height_,
8342 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8343 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8344 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8345
8346 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008347 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008348
8349 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8350
8351 VideoEncoderConfig video_encoder_config;
8352 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8353 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8354 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8355 MinEncBitrateKbps * 1000;
8356 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8357 kMaxPayloadLength);
8358
8359 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8360 WaitForEncodedFrame(1);
8361 EXPECT_EQ(
8362 MaxEncBitrateKbps,
8363 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8364 EXPECT_EQ(
8365 MinEncBitrateKbps,
8366 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8367
8368 video_stream_encoder_->Stop();
8369}
8370
8371TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8372 // Set QP trusted in encoder info.
8373 fake_encoder_.SetIsQpTrusted(false);
8374
8375 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8376 EncoderInfoSettings::
8377 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8378 codec_width_ * codec_height_,
8379 EncoderInfoSettings::
8380 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8381 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8382
8383 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8384 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8385 const int TargetEncBitrate = MaxEncBitrate;
8386 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8387 DataRate::BitsPerSec(TargetEncBitrate),
8388 DataRate::BitsPerSec(TargetEncBitrate),
8389 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8390
8391 VideoEncoderConfig video_encoder_config;
8392 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8393 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8394 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8395 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8396 kMaxPayloadLength);
8397
8398 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8399 WaitForEncodedFrame(1);
8400 EXPECT_EQ(
8401 MaxEncBitrate / 1000,
8402 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8403 EXPECT_EQ(
8404 MinEncBitrate / 1000,
8405 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8406
8407 video_stream_encoder_->Stop();
8408}
8409
Erik Språnge4589cb2022-04-06 16:44:30 +02008410TEST_F(VideoStreamEncoderTest, NormalComplexityWithMoreThanTwoCores) {
8411 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8412 /*num_spatial_layers=*/1,
8413 /*screenshare=*/false, /*allocation_callback_type=*/
8414 VideoStreamEncoder::BitrateAllocationCallbackType::
8415 kVideoBitrateAllocationWhenScreenSharing,
8416 /*num_cores=*/3);
8417
8418 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8419 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8420 video_source_.IncomingCapturedFrame(
8421 CreateFrame(1, /*width=*/320, /*height=*/180));
8422 WaitForEncodedFrame(1);
8423 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8424 VideoCodecComplexity::kComplexityNormal);
8425 video_stream_encoder_->Stop();
8426}
8427
8428TEST_F(VideoStreamEncoderTest,
8429 NormalComplexityWhenLowTierOptimizationsAreDisabled) {
8430 webrtc::test::ScopedKeyValueConfig field_trials(
8431 field_trials_, "WebRTC-VP9-LowTierOptimizations/Disabled/");
8432
8433 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8434 /*num_spatial_layers=*/1,
8435 /*screenshare=*/false, /*allocation_callback_type=*/
8436 VideoStreamEncoder::BitrateAllocationCallbackType::
8437 kVideoBitrateAllocationWhenScreenSharing,
8438 /*num_cores=*/2);
8439
8440 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8441 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8442 video_source_.IncomingCapturedFrame(
8443 CreateFrame(1, /*width=*/320, /*height=*/180));
8444 WaitForEncodedFrame(1);
8445 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8446 VideoCodecComplexity::kComplexityNormal);
8447 video_stream_encoder_->Stop();
8448}
8449
8450TEST_F(VideoStreamEncoderTest, LowComplexityWithTwoCores) {
8451 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8452 /*num_spatial_layers=*/1,
8453 /*screenshare=*/false, /*allocation_callback_type=*/
8454 VideoStreamEncoder::BitrateAllocationCallbackType::
8455 kVideoBitrateAllocationWhenScreenSharing,
8456 /*num_cores=*/2);
8457
8458 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8459 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8460 video_source_.IncomingCapturedFrame(
8461 CreateFrame(1, /*width=*/320, /*height=*/180));
8462 WaitForEncodedFrame(1);
8463 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8464 VideoCodecComplexity::kComplexityLow);
8465 video_stream_encoder_->Stop();
8466}
8467
Sergey Silkind19e3b92021-03-16 10:05:30 +00008468#if !defined(WEBRTC_IOS)
8469// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8470// disabled by default on iOS.
8471TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8472 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8473
8474 // Disable scaling settings in encoder info.
8475 fake_encoder_.SetQualityScaling(false);
8476 // Enable quality scaling in encoder config.
8477 video_encoder_config.is_quality_scaling_allowed = true;
8478 ConfigureEncoder(std::move(video_encoder_config));
8479
8480 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008481 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008482
8483 test::FrameForwarder source;
8484 video_stream_encoder_->SetSource(
8485 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8486 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8487 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8488
8489 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8490 WaitForEncodedFrame(1);
8491 video_stream_encoder_->TriggerQualityLow();
8492 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8493
8494 video_stream_encoder_->Stop();
8495}
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008496
8497TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8498 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8499
8500 // Disable scaling settings in encoder info.
8501 fake_encoder_.SetQualityScaling(false);
8502 // Set QP trusted in encoder info.
8503 fake_encoder_.SetIsQpTrusted(true);
8504 // Enable quality scaling in encoder config.
8505 video_encoder_config.is_quality_scaling_allowed = true;
8506 ConfigureEncoder(std::move(video_encoder_config));
8507
8508 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008509 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008510
8511 test::FrameForwarder source;
8512 video_stream_encoder_->SetSource(
8513 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8514 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8515 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8516
8517 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8518 WaitForEncodedFrame(1);
8519 video_stream_encoder_->TriggerQualityLow();
8520 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8521
8522 video_stream_encoder_->Stop();
8523}
Shuhai Pengf2707702021-09-29 17:19:44 +08008524
8525TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8526 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8527
8528 // Disable scaling settings in encoder info.
8529 fake_encoder_.SetQualityScaling(false);
8530 // Set QP not trusted in encoder info.
8531 fake_encoder_.SetIsQpTrusted(false);
8532 // Enable quality scaling in encoder config.
8533 video_encoder_config.is_quality_scaling_allowed = true;
8534 ConfigureEncoder(std::move(video_encoder_config));
8535
8536 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008537 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008538
8539 test::FrameForwarder source;
8540 video_stream_encoder_->SetSource(
8541 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8542 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8543 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8544
8545 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8546 WaitForEncodedFrame(1);
8547 video_stream_encoder_->TriggerQualityLow();
8548 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8549 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8550 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8551
8552 video_stream_encoder_->Stop();
8553}
8554
8555TEST_F(VideoStreamEncoderTest,
8556 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8557 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8558
8559 // Disable scaling settings in encoder info.
8560 fake_encoder_.SetQualityScaling(false);
8561 // Set QP trusted in encoder info.
8562 fake_encoder_.SetIsQpTrusted(true);
8563 // Enable quality scaling in encoder config.
8564 video_encoder_config.is_quality_scaling_allowed = true;
8565 ConfigureEncoder(std::move(video_encoder_config));
8566
8567 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008568 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008569
8570 test::FrameForwarder source;
8571 video_stream_encoder_->SetSource(
8572 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8573 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8574 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8575
8576 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8577 WaitForEncodedFrame(1);
8578 video_stream_encoder_->TriggerQualityLow();
8579 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8580 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8581
8582 video_stream_encoder_->Stop();
8583}
8584
8585TEST_F(VideoStreamEncoderTest,
8586 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8587 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8588
8589 // Disable scaling settings in encoder info.
8590 fake_encoder_.SetQualityScaling(false);
8591 // Set QP trusted in encoder info.
8592 fake_encoder_.SetIsQpTrusted(false);
8593 // Enable quality scaling in encoder config.
8594 video_encoder_config.is_quality_scaling_allowed = true;
8595 ConfigureEncoder(std::move(video_encoder_config));
8596
8597 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008598 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008599
8600 test::FrameForwarder source;
8601 video_stream_encoder_->SetSource(
8602 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8603 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8604 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8605
8606 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8607 WaitForEncodedFrame(1);
8608 video_stream_encoder_->TriggerQualityLow();
8609 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8610
8611 video_stream_encoder_->Stop();
8612}
8613
Erik Språnge4589cb2022-04-06 16:44:30 +02008614#endif // !defined(WEBRTC_IOS)
Sergey Silkind19e3b92021-03-16 10:05:30 +00008615
Henrik Boström56db9ff2021-03-24 09:06:45 +01008616// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8617class VideoStreamEncoderWithRealEncoderTest
8618 : public VideoStreamEncoderTest,
8619 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8620 public:
8621 VideoStreamEncoderWithRealEncoderTest()
8622 : VideoStreamEncoderTest(),
8623 codec_type_(std::get<0>(GetParam())),
8624 allow_i420_conversion_(std::get<1>(GetParam())) {}
8625
8626 void SetUp() override {
8627 VideoStreamEncoderTest::SetUp();
8628 std::unique_ptr<VideoEncoder> encoder;
8629 switch (codec_type_) {
8630 case kVideoCodecVP8:
8631 encoder = VP8Encoder::Create();
8632 break;
8633 case kVideoCodecVP9:
8634 encoder = VP9Encoder::Create();
8635 break;
8636 case kVideoCodecAV1:
philipel09a28482022-05-25 09:47:06 +02008637 encoder = CreateLibaomAv1Encoder();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008638 break;
8639 case kVideoCodecH264:
8640 encoder =
8641 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8642 break;
8643 case kVideoCodecMultiplex:
8644 mock_encoder_factory_for_multiplex_ =
8645 std::make_unique<MockVideoEncoderFactory>();
8646 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8647 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8648 .WillRepeatedly([] { return VP8Encoder::Create(); });
8649 encoder = std::make_unique<MultiplexEncoderAdapter>(
8650 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8651 false);
8652 break;
8653 default:
Artem Titovd3251962021-11-15 16:57:07 +01008654 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008655 }
8656 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8657 }
8658
8659 void TearDown() override {
8660 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 16:22:11 +02008661 // Ensure `video_stream_encoder_` is destroyed before
8662 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 09:06:45 +01008663 video_stream_encoder_.reset();
8664 VideoStreamEncoderTest::TearDown();
8665 }
8666
8667 protected:
8668 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8669 std::unique_ptr<VideoEncoder> encoder) {
8670 // Configure VSE to use the encoder.
8671 encoder_ = std::move(encoder);
8672 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8673 encoder_.get(), &encoder_selector_);
8674 video_send_config_.encoder_settings.encoder_factory =
8675 encoder_proxy_factory_.get();
8676 VideoEncoderConfig video_encoder_config;
8677 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8678 video_encoder_config_ = video_encoder_config.Copy();
8679 ConfigureEncoder(video_encoder_config_.Copy());
8680
8681 // Set bitrate to ensure frame is not dropped.
8682 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008683 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008684 }
8685
8686 const VideoCodecType codec_type_;
8687 const bool allow_i420_conversion_;
8688 NiceMock<MockEncoderSelector> encoder_selector_;
8689 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8690 std::unique_ptr<VideoEncoder> encoder_;
8691 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8692};
8693
8694TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8695 auto native_i420_frame = test::CreateMappableNativeFrame(
8696 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8697 video_source_.IncomingCapturedFrame(native_i420_frame);
8698 WaitForEncodedFrame(codec_width_, codec_height_);
8699
8700 auto mappable_native_buffer =
8701 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8702 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8703 mappable_native_buffer->GetMappedFramedBuffers();
8704 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8705 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8706 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8707 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8708}
8709
8710TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8711 auto native_nv12_frame = test::CreateMappableNativeFrame(
8712 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8713 video_source_.IncomingCapturedFrame(native_nv12_frame);
8714 WaitForEncodedFrame(codec_width_, codec_height_);
8715
8716 auto mappable_native_buffer =
8717 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8718 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8719 mappable_native_buffer->GetMappedFramedBuffers();
8720 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8721 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8722 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8723 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8724
8725 if (!allow_i420_conversion_) {
8726 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8727 }
8728}
8729
Erik Språng7444b192021-06-02 14:02:13 +02008730TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8731 if (codec_type_ == kVideoCodecMultiplex) {
8732 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8733 return;
8734 }
8735
8736 const size_t kNumSpatialLayers = 3u;
8737 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8738 const int kFrameWidth = 1280;
8739 const int kFrameHeight = 720;
8740 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8741 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8742 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8743 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8744 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8745 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8746
8747 VideoEncoderConfig config;
8748 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8749 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008750 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008751 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8752 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8753 vp9_settings.numberOfTemporalLayers = 3;
8754 vp9_settings.automaticResizeOn = false;
8755 config.encoder_specific_settings =
8756 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8757 vp9_settings);
8758 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8759 /*fps=*/30.0,
8760 /*first_active_layer=*/0,
8761 /*num_spatial_layers=*/3,
8762 /*num_temporal_layers=*/3,
8763 /*is_screenshare=*/false);
8764 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8765 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008766 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008767 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8768 /*fps=*/30.0,
8769 /*first_active_layer=*/0,
8770 /*num_spatial_layers=*/3,
8771 /*num_temporal_layers=*/3,
8772 /*is_screenshare=*/false);
Niels Möller79d566b2022-04-29 11:03:13 +02008773 config.simulcast_layers[0].scalability_mode = ScalabilityMode::kL3T3_KEY;
Erik Språng7444b192021-06-02 14:02:13 +02008774 } else {
8775 // Simulcast for VP8/H264.
8776 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8777 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8778 config.simulcast_layers[i].scale_resolution_down_by =
8779 kDownscaleFactors[i];
8780 config.simulcast_layers[i].active = true;
8781 }
8782 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8783 // Turn off frame dropping to prevent flakiness.
Niels Möller807328f2022-05-12 16:16:39 +02008784 config.frame_drop_enabled = false;
Erik Språng7444b192021-06-02 14:02:13 +02008785 }
8786 }
8787
8788 auto set_layer_active = [&](int layer_idx, bool active) {
8789 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8790 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8791 config.spatial_layers[layer_idx].active = active;
8792 } else {
8793 config.simulcast_layers[layer_idx].active = active;
8794 }
8795 };
8796
8797 config.video_stream_factory =
8798 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8799 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8800 /*screencast*/ false,
8801 /*screenshare enabled*/ false);
8802 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008803 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8804 0, 0, 0);
Erik Språng7444b192021-06-02 14:02:13 +02008805
8806 // Capture a frame with all layers active.
8807 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8808 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8809 int64_t timestamp_ms = kFrameIntervalMs;
8810 video_source_.IncomingCapturedFrame(
8811 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8812
8813 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8814 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8815
8816 // Capture a frame with one of the layers inactive.
8817 set_layer_active(2, false);
8818 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8819 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8820 timestamp_ms += kFrameIntervalMs;
8821 video_source_.IncomingCapturedFrame(
8822 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8823 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8824
8825 // New target bitrates signaled based on lower resolution.
8826 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8827 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8828 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8829 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8830
8831 // Re-enable the top layer.
8832 set_layer_active(2, true);
8833 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8834 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8835 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8836
8837 // Bitrate target adjusted back up to enable HD layer...
8838 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8839 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8840 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8841 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8842
8843 // ...then add a new frame.
8844 timestamp_ms += kFrameIntervalMs;
8845 video_source_.IncomingCapturedFrame(
8846 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8847 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8848 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8849
8850 video_stream_encoder_->Stop();
8851}
8852
Henrik Boström56db9ff2021-03-24 09:06:45 +01008853std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8854 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8855 VideoCodecType codec_type = std::get<0>(info.param);
8856 bool allow_i420_conversion = std::get<1>(info.param);
8857 std::string str;
8858 switch (codec_type) {
8859 case kVideoCodecGeneric:
8860 str = "Generic";
8861 break;
8862 case kVideoCodecVP8:
8863 str = "VP8";
8864 break;
8865 case kVideoCodecVP9:
8866 str = "VP9";
8867 break;
8868 case kVideoCodecAV1:
8869 str = "AV1";
8870 break;
8871 case kVideoCodecH264:
8872 str = "H264";
8873 break;
8874 case kVideoCodecMultiplex:
8875 str = "Multiplex";
8876 break;
8877 default:
Artem Titovd3251962021-11-15 16:57:07 +01008878 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008879 }
8880 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8881 return str;
8882}
8883
8884constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8885 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8886constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8887 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8888constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
Ilya Nikolaevskiy1bcdafc2022-03-09 16:01:07 +01008889 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/false);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008890constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8891 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8892#if defined(WEBRTC_USE_H264)
8893constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8894 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8895
8896// The windows compiler does not tolerate #if statements inside the
8897// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8898// and without H264).
8899INSTANTIATE_TEST_SUITE_P(
8900 All,
8901 VideoStreamEncoderWithRealEncoderTest,
8902 ::testing::Values(kVP8DisallowConversion,
8903 kVP9DisallowConversion,
8904 kAV1AllowConversion,
8905 kMultiplexDisallowConversion,
8906 kH264AllowConversion),
8907 TestParametersVideoCodecAndAllowI420ConversionToString);
8908#else
8909INSTANTIATE_TEST_SUITE_P(
8910 All,
8911 VideoStreamEncoderWithRealEncoderTest,
8912 ::testing::Values(kVP8DisallowConversion,
8913 kVP9DisallowConversion,
8914 kAV1AllowConversion,
8915 kMultiplexDisallowConversion),
8916 TestParametersVideoCodecAndAllowI420ConversionToString);
8917#endif
8918
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008919class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8920 protected:
8921 void RunTest(const std::vector<VideoStream>& configs,
8922 const int expected_num_init_encode) {
8923 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008924 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008925 InsertFrameAndWaitForEncoded();
8926 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8927 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008928 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8929 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008930
8931 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8932 ConfigureEncoder(configs[1]);
8933 InsertFrameAndWaitForEncoded();
8934 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8935 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008936 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008937 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 10:07:11 +02008938 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008939
8940 video_stream_encoder_->Stop();
8941 }
8942
8943 void ConfigureEncoder(const VideoStream& stream) {
8944 VideoEncoderConfig config;
8945 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8946 config.max_bitrate_bps = stream.max_bitrate_bps;
8947 config.simulcast_layers[0] = stream;
8948 config.video_stream_factory =
8949 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8950 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8951 /*conference_mode=*/false);
8952 video_stream_encoder_->ConfigureEncoder(std::move(config),
8953 kMaxPayloadLength);
8954 }
8955
8956 void OnBitrateUpdated(DataRate bitrate) {
8957 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8958 bitrate, bitrate, bitrate, 0, 0, 0);
8959 }
8960
8961 void InsertFrameAndWaitForEncoded() {
8962 timestamp_ms_ += kFrameIntervalMs;
8963 video_source_.IncomingCapturedFrame(
8964 CreateFrame(timestamp_ms_, kWidth, kHeight));
8965 sink_.WaitForEncodedFrame(timestamp_ms_);
8966 }
8967
8968 void ExpectEqual(const VideoCodec& actual,
8969 const VideoStream& expected) const {
8970 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8971 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8972 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
8973 static_cast<unsigned int>(expected.min_bitrate_bps));
8974 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
8975 static_cast<unsigned int>(expected.max_bitrate_bps));
8976 EXPECT_EQ(actual.simulcastStream[0].width,
8977 kWidth / expected.scale_resolution_down_by);
8978 EXPECT_EQ(actual.simulcastStream[0].height,
8979 kHeight / expected.scale_resolution_down_by);
8980 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
8981 expected.num_temporal_layers);
Niels Möller79d566b2022-04-29 11:03:13 +02008982 EXPECT_EQ(actual.GetScalabilityMode(), expected.scalability_mode);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008983 }
8984
8985 VideoStream DefaultConfig() const {
8986 VideoStream stream;
8987 stream.max_framerate = 25;
8988 stream.min_bitrate_bps = 35000;
8989 stream.max_bitrate_bps = 900000;
8990 stream.scale_resolution_down_by = 1.0;
8991 stream.num_temporal_layers = 1;
8992 stream.bitrate_priority = 1.0;
Niels Möller79d566b2022-04-29 11:03:13 +02008993 stream.scalability_mode = absl::nullopt;
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008994 return stream;
8995 }
8996
8997 const int kWidth = 640;
8998 const int kHeight = 360;
8999 int64_t timestamp_ms_ = 0;
9000};
9001
9002TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
9003 VideoStream config1 = DefaultConfig();
9004 VideoStream config2 = config1;
9005 config2.max_framerate++;
9006
9007 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9008}
9009
9010TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
9011 VideoStream config1 = DefaultConfig();
9012 VideoStream config2 = config1;
9013 config2.min_bitrate_bps += 10000;
9014
9015 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9016}
9017
9018TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
9019 VideoStream config1 = DefaultConfig();
9020 VideoStream config2 = config1;
9021 config2.max_bitrate_bps += 100000;
9022
9023 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9024}
9025
9026TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
9027 VideoStream config1 = DefaultConfig();
9028 VideoStream config2 = config1;
9029 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
9030
9031 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9032}
9033
9034TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
9035 VideoStream config1 = DefaultConfig();
9036 VideoStream config2 = config1;
9037 config2.scale_resolution_down_by *= 2;
9038
9039 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9040}
9041
9042TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
9043 VideoStream config1 = DefaultConfig();
9044 VideoStream config2 = config1;
9045 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
9046
9047 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9048}
9049
9050TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
9051 VideoStream config1 = DefaultConfig();
9052 VideoStream config2 = config1;
Niels Möller05954892022-05-13 13:34:37 +02009053 config2.scalability_mode = ScalabilityMode::kL2T1;
9054
9055 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9056}
9057
9058TEST_F(ReconfigureEncoderTest,
9059 UpdatesNumTemporalLayersFromScalabilityModeChanges) {
9060 VideoStream config1 = DefaultConfig();
9061 VideoStream config2 = config1;
Niels Möller79d566b2022-04-29 11:03:13 +02009062 config2.scalability_mode = ScalabilityMode::kL1T2;
Niels Möller05954892022-05-13 13:34:37 +02009063 config2.num_temporal_layers = 2;
Åsa Persson4d4f62f2021-09-12 16:14:48 +02009064
9065 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9066}
9067
Tommi62b01db2022-01-25 23:41:22 +01009068// Simple test that just creates and then immediately destroys an encoder.
9069// The purpose of the test is to make sure that nothing bad happens if the
9070// initialization step on the encoder queue, doesn't run.
9071TEST(VideoStreamEncoderSimpleTest, CreateDestroy) {
9072 class SuperLazyTaskQueue : public webrtc::TaskQueueBase {
9073 public:
9074 SuperLazyTaskQueue() = default;
9075 ~SuperLazyTaskQueue() override = default;
9076
9077 private:
9078 void Delete() override { delete this; }
9079 void PostTask(std::unique_ptr<QueuedTask> task) override {
9080 // meh.
9081 }
9082 void PostDelayedTask(std::unique_ptr<QueuedTask> task,
9083 uint32_t milliseconds) override {
9084 ASSERT_TRUE(false);
9085 }
9086 };
9087
9088 // Lots of boiler plate.
Jonas Oreland8ca06132022-03-14 12:52:48 +01009089 test::ScopedKeyValueConfig field_trials;
Tommi62b01db2022-01-25 23:41:22 +01009090 GlobalSimulatedTimeController time_controller(Timestamp::Millis(0));
9091 auto stats_proxy = std::make_unique<MockableSendStatisticsProxy>(
9092 time_controller.GetClock(), VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 12:52:48 +01009093 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo, field_trials);
Tommi62b01db2022-01-25 23:41:22 +01009094 SimpleVideoStreamEncoderFactory::MockFakeEncoder mock_fake_encoder(
9095 time_controller.GetClock());
9096 test::VideoEncoderProxyFactory encoder_factory(&mock_fake_encoder);
9097 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory =
9098 CreateBuiltinVideoBitrateAllocatorFactory();
9099 VideoStreamEncoderSettings encoder_settings{
9100 VideoEncoder::Capabilities(/*loss_notification=*/false)};
9101 encoder_settings.encoder_factory = &encoder_factory;
9102 encoder_settings.bitrate_allocator_factory = bitrate_allocator_factory.get();
9103
9104 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9105 EXPECT_CALL((*adapter.get()), Initialize).WillOnce(Return());
9106
9107 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
9108 encoder_queue(new SuperLazyTaskQueue());
9109
9110 // Construct a VideoStreamEncoder instance and let it go out of scope without
9111 // doing anything else (including calling Stop()). This should be fine since
9112 // the posted init task will simply be deleted.
9113 auto encoder = std::make_unique<VideoStreamEncoder>(
9114 time_controller.GetClock(), 1, stats_proxy.get(), encoder_settings,
Markus Handell8e4197b2022-05-30 15:45:28 +02009115 std::make_unique<CpuOveruseDetectorProxy>(stats_proxy.get(),
9116 field_trials),
Tommi62b01db2022-01-25 23:41:22 +01009117 std::move(adapter), std::move(encoder_queue),
9118 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009119 kVideoBitrateAllocation,
9120 field_trials);
Tommi4d6dd172022-05-23 09:34:41 +02009121
9122 // Stop the encoder explicitly. This additional step tests if we could
9123 // hang when calling stop and the TQ has been stopped and/or isn't accepting
9124 // any more tasks.
9125 encoder->Stop();
Tommi62b01db2022-01-25 23:41:22 +01009126}
9127
Markus Handellb4e96d42021-11-05 12:00:55 +01009128TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
9129 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9130 auto* adapter_ptr = adapter.get();
9131 SimpleVideoStreamEncoderFactory factory;
Markus Handell8d87c462021-12-16 11:37:16 +01009132 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9133 nullptr;
9134 EXPECT_CALL(*adapter_ptr, Initialize)
9135 .WillOnce(Invoke([&video_stream_encoder_callback](
9136 FrameCadenceAdapterInterface::Callback* callback) {
9137 video_stream_encoder_callback = callback;
9138 }));
9139 TaskQueueBase* encoder_queue = nullptr;
9140 auto video_stream_encoder =
9141 factory.Create(std::move(adapter), &encoder_queue);
Markus Handellb4e96d42021-11-05 12:00:55 +01009142
Markus Handelle59fee82021-12-23 09:29:23 +01009143 // First a call before we know the frame size and hence cannot compute the
9144 // number of simulcast layers.
9145 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
9146 &FrameCadenceAdapterInterface::
9147 ZeroHertzModeParams::num_simulcast_layers,
9148 Eq(0)))));
Markus Handellb4e96d42021-11-05 12:00:55 +01009149 VideoEncoderConfig config;
Markus Handell8d87c462021-12-16 11:37:16 +01009150 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
Markus Handellb4e96d42021-11-05 12:00:55 +01009151 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9152 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
Markus Handelle59fee82021-12-23 09:29:23 +01009153 factory.DepleteTaskQueues();
9154
9155 // Then a call as we've computed the number of simulcast layers after a passed
9156 // frame.
9157 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
9158 &FrameCadenceAdapterInterface::
9159 ZeroHertzModeParams::num_simulcast_layers,
9160 Gt(0)))));
Markus Handell8d87c462021-12-16 11:37:16 +01009161 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handell9a478b52021-11-18 16:07:01 +01009162 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01009163 Mock::VerifyAndClearExpectations(adapter_ptr);
9164
Markus Handelle59fee82021-12-23 09:29:23 +01009165 // Expect a disabled zero-hertz mode after passing realtime video.
Markus Handell8d87c462021-12-16 11:37:16 +01009166 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Eq(absl::nullopt)));
Markus Handellb4e96d42021-11-05 12:00:55 +01009167 VideoEncoderConfig config2;
Markus Handell8d87c462021-12-16 11:37:16 +01009168 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config2);
Markus Handellb4e96d42021-11-05 12:00:55 +01009169 config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
9170 video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
Markus Handell8d87c462021-12-16 11:37:16 +01009171 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
Markus Handell9a478b52021-11-18 16:07:01 +01009172 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01009173}
9174
9175TEST(VideoStreamEncoderFrameCadenceTest,
9176 ForwardsFramesIntoFrameCadenceAdapter) {
9177 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9178 auto* adapter_ptr = adapter.get();
9179 test::FrameForwarder video_source;
9180 SimpleVideoStreamEncoderFactory factory;
9181 auto video_stream_encoder = factory.Create(std::move(adapter));
9182 video_stream_encoder->SetSource(
9183 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9184
9185 EXPECT_CALL(*adapter_ptr, OnFrame);
9186 auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
9187 video_source.IncomingCapturedFrame(
Markus Handell8d87c462021-12-16 11:37:16 +01009188 VideoFrame::Builder().set_video_frame_buffer(std::move(buffer)).build());
Markus Handellb4e96d42021-11-05 12:00:55 +01009189}
9190
Markus Handellee225432021-11-29 12:35:12 +01009191TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) {
9192 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9193 auto* adapter_ptr = adapter.get();
9194 test::FrameForwarder video_source;
9195 SimpleVideoStreamEncoderFactory factory;
9196 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9197 nullptr;
9198 EXPECT_CALL(*adapter_ptr, Initialize)
9199 .WillOnce(Invoke([&video_stream_encoder_callback](
9200 FrameCadenceAdapterInterface::Callback* callback) {
9201 video_stream_encoder_callback = callback;
9202 }));
9203 TaskQueueBase* encoder_queue = nullptr;
9204 auto video_stream_encoder =
9205 factory.Create(std::move(adapter), &encoder_queue);
9206
9207 // This is just to make the VSE operational. We'll feed a frame directly by
9208 // the callback interface.
9209 video_stream_encoder->SetSource(
9210 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9211
9212 VideoEncoderConfig video_encoder_config;
9213 test::FillEncoderConfiguration(kVideoCodecGeneric, 1, &video_encoder_config);
9214 video_stream_encoder->ConfigureEncoder(std::move(video_encoder_config),
9215 /*max_data_payload_length=*/1000);
9216
9217 EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps);
9218 EXPECT_CALL(*adapter_ptr, UpdateFrameRate);
Markus Handell8d87c462021-12-16 11:37:16 +01009219 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handellee225432021-11-29 12:35:12 +01009220 factory.DepleteTaskQueues();
9221}
9222
Markus Handell8d87c462021-12-16 11:37:16 +01009223TEST(VideoStreamEncoderFrameCadenceTest,
9224 DeactivatesActivatesLayersOnBitrateChanges) {
9225 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9226 auto* adapter_ptr = adapter.get();
9227 SimpleVideoStreamEncoderFactory factory;
9228 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9229 nullptr;
9230 EXPECT_CALL(*adapter_ptr, Initialize)
9231 .WillOnce(Invoke([&video_stream_encoder_callback](
9232 FrameCadenceAdapterInterface::Callback* callback) {
9233 video_stream_encoder_callback = callback;
9234 }));
9235 TaskQueueBase* encoder_queue = nullptr;
9236 auto video_stream_encoder =
9237 factory.Create(std::move(adapter), &encoder_queue);
9238
9239 // Configure 2 simulcast layers. FillEncoderConfiguration sets min bitrates to
9240 // {150000, 450000}.
9241 VideoEncoderConfig video_encoder_config;
9242 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9243 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9244 kMaxPayloadLength);
9245 // Ensure an encoder is created.
9246 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9247
9248 // Both layers enabled at 1 MBit/s.
9249 video_stream_encoder->OnBitrateUpdated(
9250 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9251 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9252 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9253 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9254 factory.DepleteTaskQueues();
9255 Mock::VerifyAndClearExpectations(adapter_ptr);
9256
9257 // Layer 1 disabled at 200 KBit/s.
9258 video_stream_encoder->OnBitrateUpdated(
9259 DataRate::KilobitsPerSec(200), DataRate::KilobitsPerSec(200),
9260 DataRate::KilobitsPerSec(200), 0, 0, 0);
9261 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9262 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9263 factory.DepleteTaskQueues();
9264 Mock::VerifyAndClearExpectations(adapter_ptr);
9265
9266 // All layers off at suspended video.
9267 video_stream_encoder->OnBitrateUpdated(DataRate::Zero(), DataRate::Zero(),
9268 DataRate::Zero(), 0, 0, 0);
9269 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/false));
9270 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9271 factory.DepleteTaskQueues();
9272 Mock::VerifyAndClearExpectations(adapter_ptr);
9273
9274 // Both layers enabled again back at 1 MBit/s.
9275 video_stream_encoder->OnBitrateUpdated(
9276 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9277 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9278 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9279 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9280 factory.DepleteTaskQueues();
9281}
9282
9283TEST(VideoStreamEncoderFrameCadenceTest, UpdatesQualityConvergence) {
9284 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9285 auto* adapter_ptr = adapter.get();
9286 SimpleVideoStreamEncoderFactory factory;
9287 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9288 nullptr;
9289 EXPECT_CALL(*adapter_ptr, Initialize)
9290 .WillOnce(Invoke([&video_stream_encoder_callback](
9291 FrameCadenceAdapterInterface::Callback* callback) {
9292 video_stream_encoder_callback = callback;
9293 }));
9294 TaskQueueBase* encoder_queue = nullptr;
9295 auto video_stream_encoder =
9296 factory.Create(std::move(adapter), &encoder_queue);
9297
9298 // Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder.
9299 VideoEncoderConfig video_encoder_config;
9300 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9301 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9302 kMaxPayloadLength);
9303 video_stream_encoder->OnBitrateUpdated(
9304 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9305 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9306
9307 // Pass a frame which has unconverged results.
9308 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9309 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9310 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9311 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
9312 EXPECT_FALSE(encoded_image.IsAtTargetQuality());
9313 CodecSpecificInfo codec_specific;
9314 codec_specific.codecType = kVideoCodecGeneric;
9315 return codec_specific;
9316 }));
9317 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, false));
9318 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9319 factory.DepleteTaskQueues();
9320 Mock::VerifyAndClearExpectations(adapter_ptr);
9321 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9322
9323 // Pass a frame which converges in layer 0 and not in layer 1.
9324 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9325 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9326 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9327 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
9328 encoded_image.SetAtTargetQuality(encoded_image.SpatialIndex() == 0);
9329 CodecSpecificInfo codec_specific;
9330 codec_specific.codecType = kVideoCodecGeneric;
9331 return codec_specific;
9332 }));
9333 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, true));
9334 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9335 factory.DepleteTaskQueues();
9336 Mock::VerifyAndClearExpectations(adapter_ptr);
9337 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9338}
9339
Markus Handell2e0f4f02021-12-21 19:14:58 +01009340TEST(VideoStreamEncoderFrameCadenceTest,
9341 RequestsRefreshFramesWhenCadenceAdapterInstructs) {
9342 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9343 auto* adapter_ptr = adapter.get();
9344 MockVideoSourceInterface mock_source;
9345 SimpleVideoStreamEncoderFactory factory;
9346 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9347 nullptr;
9348 EXPECT_CALL(*adapter_ptr, Initialize)
9349 .WillOnce(Invoke([&video_stream_encoder_callback](
9350 FrameCadenceAdapterInterface::Callback* callback) {
9351 video_stream_encoder_callback = callback;
9352 }));
9353 TaskQueueBase* encoder_queue = nullptr;
9354 auto video_stream_encoder =
9355 factory.Create(std::move(adapter), &encoder_queue);
9356 video_stream_encoder->SetSource(
9357 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9358 VideoEncoderConfig config;
9359 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9360 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9361 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9362 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9363 // Ensure the encoder is set up.
9364 factory.DepleteTaskQueues();
9365
Markus Handell818e7fb2021-12-30 13:01:33 +01009366 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest)
9367 .WillOnce(Invoke([video_stream_encoder_callback] {
9368 video_stream_encoder_callback->RequestRefreshFrame();
9369 }));
Markus Handell2e0f4f02021-12-21 19:14:58 +01009370 EXPECT_CALL(mock_source, RequestRefreshFrame);
9371 video_stream_encoder->SendKeyFrame();
9372 factory.DepleteTaskQueues();
9373 Mock::VerifyAndClearExpectations(adapter_ptr);
9374 Mock::VerifyAndClearExpectations(&mock_source);
9375
Markus Handell818e7fb2021-12-30 13:01:33 +01009376 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest);
Markus Handell2e0f4f02021-12-21 19:14:58 +01009377 EXPECT_CALL(mock_source, RequestRefreshFrame).Times(0);
9378 video_stream_encoder->SendKeyFrame();
9379 factory.DepleteTaskQueues();
9380}
9381
Markus Handell818e7fb2021-12-30 13:01:33 +01009382TEST(VideoStreamEncoderFrameCadenceTest,
9383 RequestsRefreshFrameForEarlyZeroHertzKeyFrameRequest) {
9384 SimpleVideoStreamEncoderFactory factory;
9385 auto encoder_queue =
9386 factory.GetTimeController()->GetTaskQueueFactory()->CreateTaskQueue(
9387 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
9388
9389 // Enables zero-hertz mode.
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009390 test::ScopedKeyValueConfig field_trials(
9391 "WebRTC-ZeroHertzScreenshare/Enabled/");
Markus Handell818e7fb2021-12-30 13:01:33 +01009392 auto adapter = FrameCadenceAdapterInterface::Create(
Jonas Oreland8ca06132022-03-14 12:52:48 +01009393 factory.GetTimeController()->GetClock(), encoder_queue.get(),
9394 field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009395 FrameCadenceAdapterInterface* adapter_ptr = adapter.get();
9396
9397 MockVideoSourceInterface mock_source;
9398 auto video_stream_encoder = factory.CreateWithEncoderQueue(
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009399 std::move(adapter), std::move(encoder_queue), &field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009400
9401 video_stream_encoder->SetSource(
9402 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9403 VideoEncoderConfig config;
9404 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9405 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9406 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9407
9408 // Eventually expect a refresh frame request when requesting a key frame
9409 // before initializing zero-hertz mode. This can happen in reality because the
9410 // threads invoking key frame requests and constraints setup aren't
9411 // synchronized.
9412 EXPECT_CALL(mock_source, RequestRefreshFrame);
9413 video_stream_encoder->SendKeyFrame();
Markus Handellf5a50792022-06-16 14:25:15 +02009414 constexpr int kMaxFps = 30;
9415 adapter_ptr->OnConstraintsChanged(VideoTrackSourceConstraints{0, kMaxFps});
9416 factory.GetTimeController()->AdvanceTime(
9417 TimeDelta::Seconds(1) *
9418 FrameCadenceAdapterInterface::kOnDiscardedFrameRefreshFramePeriod /
9419 kMaxFps);
Markus Handell818e7fb2021-12-30 13:01:33 +01009420}
9421
perkj26091b12016-09-01 01:17:40 -07009422} // namespace webrtc