blob: 961f59216524090ff269455831303bebaa9bfdd3 [file] [log] [blame]
Jonas Oreland1262eb52022-09-27 16:53:04 +02001
perkj26091b12016-09-01 01:17:40 -07002/*
3 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
4 *
5 * Use of this source code is governed by a BSD-style license
6 * that can be found in the LICENSE file in the root of the source
7 * tree. An additional intellectual property rights grant can be found
8 * in the file PATENTS. All contributing project authors may
9 * be found in the AUTHORS file in the root of the source tree.
10 */
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Markus Handell8e4197b2022-05-30 15:45:28 +020020#include "api/field_trials_view.h"
Markus Handell9a478b52021-11-18 16:07:01 +010021#include "api/rtp_parameters.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020022#include "api/task_queue/default_task_queue_factory.h"
Markus Handell818e7fb2021-12-30 13:01:33 +010023#include "api/task_queue/task_queue_base.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010024#include "api/task_queue/task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020025#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010026#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010027#include "api/test/mock_video_encoder_factory.h"
Markus Handell8d87c462021-12-16 11:37:16 +010028#include "api/units/data_rate.h"
Markus Handellf5a50792022-06-16 14:25:15 +020029#include "api/units/time_delta.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080030#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020031#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020032#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020033#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010034#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010035#include "api/video_codecs/sdp_video_format.h"
Erik Språnge4589cb2022-04-06 16:44:30 +020036#include "api/video_codecs/video_codec.h"
Elad Alon370f93a2019-06-11 14:57:57 +020037#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020038#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010039#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020040#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010041#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020042#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070043#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080044#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020045#include "media/engine/webrtc_video_engine.h"
philipel09a28482022-05-25 09:47:06 +020046#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010047#include "modules/video_coding/codecs/h264/include/h264.h"
48#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
49#include "modules/video_coding/codecs/vp8/include/vp8.h"
50#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020051#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020052#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020053#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010054#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Markus Handelle1a198b2022-09-15 08:13:25 +000055#include "modules/video_coding/utility/vp8_constants.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020056#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010057#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020058#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020059#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080060#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020061#include "rtc_base/synchronization/mutex.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020062#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020063#include "test/encoder_settings.h"
64#include "test/fake_encoder.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010065#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020066#include "test/gmock.h"
67#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010068#include "test/mappable_native_buffer.h"
Jonas Orelandc7f691a2022-03-09 15:12:07 +010069#include "test/scoped_key_value_config.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020070#include "test/time_controller/simulated_time_controller.h"
Byoungchan Lee13fe3672022-04-06 10:44:42 +090071#include "test/video_encoder_nullable_proxy_factory.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020072#include "test/video_encoder_proxy_factory.h"
Jonas Oreland1262eb52022-09-27 16:53:04 +020073#include "video/config/encoder_stream_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010074#include "video/frame_cadence_adapter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020075#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070076
77namespace webrtc {
78
sprang57c2fff2017-01-16 06:24:02 -080079using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020080using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020081using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020082using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020083using ::testing::Ge;
84using ::testing::Gt;
Markus Handellee225432021-11-29 12:35:12 +010085using ::testing::Invoke;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020086using ::testing::Le;
87using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010088using ::testing::Matcher;
Markus Handellb4e96d42021-11-05 12:00:55 +010089using ::testing::Mock;
philipel9b058032020-02-10 11:30:00 +010090using ::testing::NiceMock;
Markus Handellb4e96d42021-11-05 12:00:55 +010091using ::testing::Optional;
philipel9b058032020-02-10 11:30:00 +010092using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010093using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020094using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080095
perkj803d97f2016-11-01 11:45:46 -070096namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020097const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010098const int kQpLow = 1;
99const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200100const int kMinFramerateFps = 2;
101const int kMinBalancedFramerateFps = 7;
Markus Handell2cfc1af2022-08-19 08:16:48 +0000102constexpr TimeDelta kFrameTimeout = TimeDelta::Millis(100);
asapersson5f7226f2016-11-25 04:37:00 -0800103const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 10:07:11 +0200104const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
105const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
106const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
107const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 07:02:22 -0800108const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -0700109const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200110const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +0200111const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200112const VideoEncoder::ResolutionBitrateLimits
113 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
114const VideoEncoder::ResolutionBitrateLimits
115 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800116
Asa Persson606d3cb2021-10-04 10:07:11 +0200117uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200118 0x00, 0x00, 0x03, 0x03, 0xF4,
119 0x05, 0x03, 0xC7, 0xE0, 0x1B,
120 0x41, 0x10, 0x8D, 0x00};
121
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100122const uint8_t kCodedFrameVp8Qp25[] = {
123 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
124 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
125 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
126
Markus Handell818e7fb2021-12-30 13:01:33 +0100127VideoFrame CreateSimpleNV12Frame() {
128 return VideoFrame::Builder()
129 .set_video_frame_buffer(rtc::make_ref_counted<NV12Buffer>(
130 /*width=*/16, /*height=*/16))
131 .build();
132}
133
Markus Handell8d87c462021-12-16 11:37:16 +0100134void PassAFrame(
135 TaskQueueBase* encoder_queue,
136 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback,
137 int64_t ntp_time_ms) {
Danil Chapovalov95eeaa72022-07-06 10:14:29 +0200138 encoder_queue->PostTask([video_stream_encoder_callback, ntp_time_ms] {
139 video_stream_encoder_callback->OnFrame(Timestamp::Millis(ntp_time_ms), 1,
140 CreateSimpleNV12Frame());
141 });
Markus Handell8d87c462021-12-16 11:37:16 +0100142}
143
perkj803d97f2016-11-01 11:45:46 -0700144class TestBuffer : public webrtc::I420Buffer {
145 public:
146 TestBuffer(rtc::Event* event, int width, int height)
147 : I420Buffer(width, height), event_(event) {}
148
149 private:
150 friend class rtc::RefCountedObject<TestBuffer>;
151 ~TestBuffer() override {
152 if (event_)
153 event_->Set();
154 }
155 rtc::Event* const event_;
156};
157
Henrik Boström56db9ff2021-03-24 09:06:45 +0100158// A fake native buffer that can't be converted to I420. Upon scaling, it
159// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700160class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
161 public:
162 FakeNativeBuffer(rtc::Event* event, int width, int height)
163 : event_(event), width_(width), height_(height) {}
164 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
165 int width() const override { return width_; }
166 int height() const override { return height_; }
167 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
168 return nullptr;
169 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100170 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
171 int offset_x,
172 int offset_y,
173 int crop_width,
174 int crop_height,
175 int scaled_width,
176 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200177 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
178 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100179 }
Noah Richards51db4212019-06-12 06:59:12 -0700180
181 private:
182 friend class rtc::RefCountedObject<FakeNativeBuffer>;
183 ~FakeNativeBuffer() override {
184 if (event_)
185 event_->Set();
186 }
187 rtc::Event* const event_;
188 const int width_;
189 const int height_;
190};
191
Evan Shrubsole895556e2020-10-05 09:15:13 +0200192// A fake native buffer that is backed by an NV12 buffer.
193class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
194 public:
195 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
196 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
197
198 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
199 int width() const override { return nv12_buffer_->width(); }
200 int height() const override { return nv12_buffer_->height(); }
201 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
202 return nv12_buffer_->ToI420();
203 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200204 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
205 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
206 if (absl::c_find(types, Type::kNV12) != types.end()) {
207 return nv12_buffer_;
208 }
209 return nullptr;
210 }
Niels Möllerba2de582022-04-20 16:46:26 +0200211 const NV12BufferInterface* GetNV12() const { return nv12_buffer_.get(); }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200212
213 private:
214 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
215 ~FakeNV12NativeBuffer() override {
216 if (event_)
217 event_->Set();
218 }
219 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
220 rtc::Event* const event_;
221};
222
Niels Möller7dc26b72017-12-06 10:27:48 +0100223class CpuOveruseDetectorProxy : public OveruseFrameDetector {
224 public:
Markus Handell8e4197b2022-05-30 15:45:28 +0200225 CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer,
226 const FieldTrialsView& field_trials)
227 : OveruseFrameDetector(metrics_observer, field_trials),
Henrik Boström381d1092020-05-12 18:49:07 +0200228 last_target_framerate_fps_(-1),
229 framerate_updated_event_(true /* manual_reset */,
230 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100231 virtual ~CpuOveruseDetectorProxy() {}
232
233 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200234 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100235 last_target_framerate_fps_ = framerate_fps;
236 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200237 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100238 }
239
240 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200241 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100242 return last_target_framerate_fps_;
243 }
244
Niels Möller4db138e2018-04-19 09:04:13 +0200245 CpuOveruseOptions GetOptions() { return options_; }
246
Henrik Boström381d1092020-05-12 18:49:07 +0200247 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
248
Niels Möller7dc26b72017-12-06 10:27:48 +0100249 private:
Markus Handella3765182020-07-08 13:13:32 +0200250 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100251 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200252 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100253};
254
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200255class FakeVideoSourceRestrictionsListener
256 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200257 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200258 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200259 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200260 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200261 RTC_DCHECK(was_restrictions_updated_);
262 }
263
264 rtc::Event* restrictions_updated_event() {
265 return &restrictions_updated_event_;
266 }
267
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200268 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200269 void OnVideoSourceRestrictionsUpdated(
270 VideoSourceRestrictions restrictions,
271 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200272 rtc::scoped_refptr<Resource> reason,
273 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200274 was_restrictions_updated_ = true;
275 restrictions_updated_event_.Set();
276 }
277
278 private:
279 bool was_restrictions_updated_;
280 rtc::Event restrictions_updated_event_;
281};
282
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200283auto WantsFps(Matcher<int> fps_matcher) {
284 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
285 fps_matcher);
286}
287
288auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
289 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
290 AllOf(max_pixel_matcher, Gt(0)));
291}
292
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200293auto ResolutionMax() {
294 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200295 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200296 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
297 Eq(absl::nullopt)));
298}
299
300auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200301 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200302}
303
304auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200305 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200306}
307
308auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200309 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200310}
311
312auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200313 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200314}
315
316auto FpsMaxResolutionMax() {
317 return AllOf(FpsMax(), ResolutionMax());
318}
319
320auto UnlimitedSinkWants() {
321 return AllOf(FpsUnlimited(), ResolutionMax());
322}
323
324auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
325 Matcher<int> fps_range_matcher;
326
327 if (last_frame_pixels <= 320 * 240) {
328 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200329 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200330 fps_range_matcher = AllOf(Ge(10), Le(15));
331 } else if (last_frame_pixels <= 640 * 480) {
332 fps_range_matcher = Ge(15);
333 } else {
334 fps_range_matcher = Eq(kDefaultFramerate);
335 }
336 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
337 fps_range_matcher);
338}
339
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200340auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
341 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
342 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
343}
344
345auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
346 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
347}
348
349auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
350 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
351}
352
353auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
354 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
355 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
356}
357
358auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
359 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
360 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
361}
362
363auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
364 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
365 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
366}
367
368auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
369 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
370 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
371}
372
mflodmancc3d4422017-08-03 08:27:51 -0700373class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700374 public:
Markus Handell9a478b52021-11-18 16:07:01 +0100375 VideoStreamEncoderUnderTest(
376 TimeController* time_controller,
377 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter,
378 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
379 encoder_queue,
380 SendStatisticsProxy* stats_proxy,
381 const VideoStreamEncoderSettings& settings,
382 VideoStreamEncoder::BitrateAllocationCallbackType
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100383 allocation_callback_type,
Erik Språnge4589cb2022-04-06 16:44:30 +0200384 const FieldTrialsView& field_trials,
385 int num_cores)
Markus Handell8e4197b2022-05-30 15:45:28 +0200386 : VideoStreamEncoder(
387 time_controller->GetClock(),
388 num_cores,
389 stats_proxy,
390 settings,
391 std::unique_ptr<OveruseFrameDetector>(
392 overuse_detector_proxy_ =
393 new CpuOveruseDetectorProxy(stats_proxy, field_trials)),
394 std::move(cadence_adapter),
395 std::move(encoder_queue),
396 allocation_callback_type,
397 field_trials),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200398 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200399 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200400 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200401 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200402 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200403 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200404 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200405 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100406 }
perkj803d97f2016-11-01 11:45:46 -0700407
Henrik Boström381d1092020-05-12 18:49:07 +0200408 void SetSourceAndWaitForRestrictionsUpdated(
409 rtc::VideoSourceInterface<VideoFrame>* source,
410 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200411 FakeVideoSourceRestrictionsListener listener;
412 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200413 SetSource(source, degradation_preference);
Markus Handell2cfc1af2022-08-19 08:16:48 +0000414 listener.restrictions_updated_event()->Wait(TimeDelta::Seconds(5));
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200415 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200416 }
417
418 void SetSourceAndWaitForFramerateUpdated(
419 rtc::VideoSourceInterface<VideoFrame>* source,
420 const DegradationPreference& degradation_preference) {
421 overuse_detector_proxy_->framerate_updated_event()->Reset();
422 SetSource(source, degradation_preference);
Markus Handell2cfc1af2022-08-19 08:16:48 +0000423 overuse_detector_proxy_->framerate_updated_event()->Wait(
424 TimeDelta::Seconds(5));
Henrik Boström381d1092020-05-12 18:49:07 +0200425 }
426
427 void OnBitrateUpdatedAndWaitForManagedResources(
428 DataRate target_bitrate,
429 DataRate stable_target_bitrate,
430 DataRate link_allocation,
431 uint8_t fraction_lost,
432 int64_t round_trip_time_ms,
433 double cwnd_reduce_ratio) {
434 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
435 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
436 // Bitrate is updated on the encoder queue.
437 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200438 }
439
kthelgason2fc52542017-03-03 00:24:41 -0800440 // This is used as a synchronisation mechanism, to make sure that the
441 // encoder queue is not blocked before we start sending it frames.
442 void WaitUntilTaskQueueIsIdle() {
Markus Handell28c71802021-11-08 10:11:55 +0100443 time_controller_->AdvanceTime(TimeDelta::Zero());
kthelgason2fc52542017-03-03 00:24:41 -0800444 }
445
Henrik Boström91aa7322020-04-28 12:24:33 +0200446 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200447 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200448 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200449 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200450 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200451 event.Set();
452 });
Markus Handell2cfc1af2022-08-19 08:16:48 +0000453 ASSERT_TRUE(event.Wait(TimeDelta::Seconds(5)));
Markus Handell28c71802021-11-08 10:11:55 +0100454 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200455 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200456
Henrik Boström91aa7322020-04-28 12:24:33 +0200457 void TriggerCpuUnderuse() {
458 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200459 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200460 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200461 event.Set();
462 });
Markus Handell2cfc1af2022-08-19 08:16:48 +0000463 ASSERT_TRUE(event.Wait(TimeDelta::Seconds(5)));
Markus Handell28c71802021-11-08 10:11:55 +0100464 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200465 }
kthelgason876222f2016-11-29 01:44:11 -0800466
Henrik Boström91aa7322020-04-28 12:24:33 +0200467 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200468 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200469 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200470 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200471 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200472 event.Set();
473 });
Markus Handell2cfc1af2022-08-19 08:16:48 +0000474 ASSERT_TRUE(event.Wait(TimeDelta::Seconds(5)));
Markus Handell28c71802021-11-08 10:11:55 +0100475 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200476 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200477 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200478 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200479 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200480 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200481 event.Set();
482 });
Markus Handell2cfc1af2022-08-19 08:16:48 +0000483 ASSERT_TRUE(event.Wait(TimeDelta::Seconds(5)));
Markus Handell28c71802021-11-08 10:11:55 +0100484 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200485 }
486
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200487 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100488 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200489 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
490 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200491 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700492};
493
Noah Richards51db4212019-06-12 06:59:12 -0700494// Simulates simulcast behavior and makes highest stream resolutions divisible
495// by 4.
496class CroppingVideoStreamFactory
497 : public VideoEncoderConfig::VideoStreamFactoryInterface {
498 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200499 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700500
501 private:
502 std::vector<VideoStream> CreateEncoderStreams(
503 int width,
504 int height,
505 const VideoEncoderConfig& encoder_config) override {
506 std::vector<VideoStream> streams = test::CreateVideoStreams(
507 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700508 return streams;
509 }
Noah Richards51db4212019-06-12 06:59:12 -0700510};
511
sprangb1ca0732017-02-01 08:38:12 -0800512class AdaptingFrameForwarder : public test::FrameForwarder {
513 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200514 explicit AdaptingFrameForwarder(TimeController* time_controller)
515 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700516 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800517
518 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200519 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800520 adaptation_enabled_ = enabled;
521 }
522
asaperssonfab67072017-04-04 05:51:49 -0700523 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200524 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800525 return adaptation_enabled_;
526 }
527
Henrik Boström1124ed12021-02-25 10:30:39 +0100528 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
529 // the resolution or frame rate was different than it is currently. If
530 // something else is modified, such as encoder resolutions, but the resolution
531 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700532 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200533 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700534 return last_wants_;
535 }
536
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200537 absl::optional<int> last_sent_width() const { return last_width_; }
538 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800539
sprangb1ca0732017-02-01 08:38:12 -0800540 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200541 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +0100542 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200543
sprangb1ca0732017-02-01 08:38:12 -0800544 int cropped_width = 0;
545 int cropped_height = 0;
546 int out_width = 0;
547 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700548 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000549 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
550 << "w=" << video_frame.width()
551 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700552 if (adapter_.AdaptFrameResolution(
553 video_frame.width(), video_frame.height(),
554 video_frame.timestamp_us() * 1000, &cropped_width,
555 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100556 VideoFrame adapted_frame =
557 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200558 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100559 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200560 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100561 .set_timestamp_ms(99)
562 .set_rotation(kVideoRotation_0)
563 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100564 if (video_frame.has_update_rect()) {
565 adapted_frame.set_update_rect(
566 video_frame.update_rect().ScaleWithFrame(
567 video_frame.width(), video_frame.height(), 0, 0,
568 video_frame.width(), video_frame.height(), out_width,
569 out_height));
570 }
sprangc5d62e22017-04-02 23:53:04 -0700571 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800572 last_width_.emplace(adapted_frame.width());
573 last_height_.emplace(adapted_frame.height());
574 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200575 last_width_ = absl::nullopt;
576 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700577 }
sprangb1ca0732017-02-01 08:38:12 -0800578 } else {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000579 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800580 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800581 last_width_.emplace(video_frame.width());
582 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800583 }
584 }
585
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200586 void OnOutputFormatRequest(int width, int height) {
587 absl::optional<std::pair<int, int>> target_aspect_ratio =
588 std::make_pair(width, height);
589 absl::optional<int> max_pixel_count = width * height;
590 absl::optional<int> max_fps;
591 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
592 max_fps);
593 }
594
sprangb1ca0732017-02-01 08:38:12 -0800595 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
596 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200597 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100598 rtc::VideoSinkWants prev_wants = sink_wants_locked();
599 bool did_adapt =
600 prev_wants.max_pixel_count != wants.max_pixel_count ||
601 prev_wants.target_pixel_count != wants.target_pixel_count ||
602 prev_wants.max_framerate_fps != wants.max_framerate_fps;
603 if (did_adapt) {
604 last_wants_ = prev_wants;
605 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100606 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200607 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800608 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200609
Erik Språng5e13d052022-08-02 11:42:49 +0200610 void RequestRefreshFrame() override { ++refresh_frames_requested_; }
611
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200612 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800613 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200614 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
615 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200616 absl::optional<int> last_width_;
617 absl::optional<int> last_height_;
Erik Språng5e13d052022-08-02 11:42:49 +0200618 int refresh_frames_requested_{0};
sprangb1ca0732017-02-01 08:38:12 -0800619};
sprangc5d62e22017-04-02 23:53:04 -0700620
Niels Möller213618e2018-07-24 09:29:58 +0200621// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700622class MockableSendStatisticsProxy : public SendStatisticsProxy {
623 public:
624 MockableSendStatisticsProxy(Clock* clock,
625 const VideoSendStream::Config& config,
Jonas Oreland8ca06132022-03-14 12:52:48 +0100626 VideoEncoderConfig::ContentType content_type,
Jonas Orelande62c2f22022-03-29 11:04:48 +0200627 const FieldTrialsView& field_trials)
Jonas Oreland8ca06132022-03-14 12:52:48 +0100628 : SendStatisticsProxy(clock, config, content_type, field_trials) {}
sprangc5d62e22017-04-02 23:53:04 -0700629
630 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200631 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700632 if (mock_stats_)
633 return *mock_stats_;
634 return SendStatisticsProxy::GetStats();
635 }
636
Niels Möller213618e2018-07-24 09:29:58 +0200637 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200638 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200639 if (mock_stats_)
640 return mock_stats_->input_frame_rate;
641 return SendStatisticsProxy::GetInputFrameRate();
642 }
sprangc5d62e22017-04-02 23:53:04 -0700643 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200644 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700645 mock_stats_.emplace(stats);
646 }
647
648 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200649 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700650 mock_stats_.reset();
651 }
652
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200653 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
654 on_frame_dropped_ = std::move(callback);
655 }
656
sprangc5d62e22017-04-02 23:53:04 -0700657 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200658 void OnFrameDropped(DropReason reason) override {
659 SendStatisticsProxy::OnFrameDropped(reason);
660 if (on_frame_dropped_)
661 on_frame_dropped_(reason);
662 }
663
Markus Handella3765182020-07-08 13:13:32 +0200664 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200665 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200666 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700667};
668
Markus Handellb4e96d42021-11-05 12:00:55 +0100669class SimpleVideoStreamEncoderFactory {
670 public:
671 class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
672 public:
673 using VideoStreamEncoder::VideoStreamEncoder;
674 ~AdaptedVideoStreamEncoder() { Stop(); }
675 };
676
Markus Handell8d87c462021-12-16 11:37:16 +0100677 class MockFakeEncoder : public test::FakeEncoder {
678 public:
679 using FakeEncoder::FakeEncoder;
680 MOCK_METHOD(CodecSpecificInfo,
681 EncodeHook,
682 (EncodedImage & encoded_image,
683 rtc::scoped_refptr<EncodedImageBuffer> buffer),
684 (override));
685 };
686
Markus Handellee225432021-11-29 12:35:12 +0100687 SimpleVideoStreamEncoderFactory() {
Markus Handellb4e96d42021-11-05 12:00:55 +0100688 encoder_settings_.encoder_factory = &encoder_factory_;
Markus Handellee225432021-11-29 12:35:12 +0100689 encoder_settings_.bitrate_allocator_factory =
690 bitrate_allocator_factory_.get();
Markus Handellb4e96d42021-11-05 12:00:55 +0100691 }
692
Markus Handell818e7fb2021-12-30 13:01:33 +0100693 std::unique_ptr<AdaptedVideoStreamEncoder> CreateWithEncoderQueue(
Markus Handellee225432021-11-29 12:35:12 +0100694 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100695 std::unique_ptr<TaskQueueBase, TaskQueueDeleter> encoder_queue,
Jonas Orelande62c2f22022-03-29 11:04:48 +0200696 const FieldTrialsView* field_trials = nullptr) {
Markus Handellb4e96d42021-11-05 12:00:55 +0100697 auto result = std::make_unique<AdaptedVideoStreamEncoder>(
698 time_controller_.GetClock(),
699 /*number_of_cores=*/1,
700 /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
Markus Handell8e4197b2022-05-30 15:45:28 +0200701 std::make_unique<CpuOveruseDetectorProxy>(
702 /*stats_proxy=*/nullptr,
703 field_trials ? *field_trials : field_trials_),
Markus Handellee225432021-11-29 12:35:12 +0100704 std::move(zero_hertz_adapter), std::move(encoder_queue),
Markus Handellb4e96d42021-11-05 12:00:55 +0100705 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100706 kVideoBitrateAllocation,
707 field_trials ? *field_trials : field_trials_);
Markus Handellb4e96d42021-11-05 12:00:55 +0100708 result->SetSink(&sink_, /*rotation_applied=*/false);
709 return result;
710 }
711
Markus Handell818e7fb2021-12-30 13:01:33 +0100712 std::unique_ptr<AdaptedVideoStreamEncoder> Create(
713 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
714 TaskQueueBase** encoder_queue_ptr = nullptr) {
715 auto encoder_queue =
716 time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
717 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
718 if (encoder_queue_ptr)
719 *encoder_queue_ptr = encoder_queue.get();
720 return CreateWithEncoderQueue(std::move(zero_hertz_adapter),
721 std::move(encoder_queue));
722 }
723
Markus Handell9a478b52021-11-18 16:07:01 +0100724 void DepleteTaskQueues() { time_controller_.AdvanceTime(TimeDelta::Zero()); }
Markus Handell8d87c462021-12-16 11:37:16 +0100725 MockFakeEncoder& GetMockFakeEncoder() { return mock_fake_encoder_; }
Markus Handell9a478b52021-11-18 16:07:01 +0100726
Markus Handell818e7fb2021-12-30 13:01:33 +0100727 GlobalSimulatedTimeController* GetTimeController() {
728 return &time_controller_;
729 }
730
Markus Handellb4e96d42021-11-05 12:00:55 +0100731 private:
732 class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
733 public:
734 ~NullEncoderSink() override = default;
735 void OnEncoderConfigurationChanged(
736 std::vector<VideoStream> streams,
737 bool is_svc,
738 VideoEncoderConfig::ContentType content_type,
739 int min_transmit_bitrate_bps) override {}
740 void OnBitrateAllocationUpdated(
741 const VideoBitrateAllocation& allocation) override {}
742 void OnVideoLayersAllocationUpdated(
743 VideoLayersAllocation allocation) override {}
744 Result OnEncodedImage(
745 const EncodedImage& encoded_image,
746 const CodecSpecificInfo* codec_specific_info) override {
747 return Result(EncodedImageCallback::Result::OK);
748 }
749 };
750
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100751 test::ScopedKeyValueConfig field_trials_;
Philipp Hanckea204ad22022-07-08 18:43:25 +0200752 GlobalSimulatedTimeController time_controller_{Timestamp::Zero()};
Markus Handellee225432021-11-29 12:35:12 +0100753 std::unique_ptr<TaskQueueFactory> task_queue_factory_{
754 time_controller_.CreateTaskQueueFactory()};
755 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_ =
756 std::make_unique<MockableSendStatisticsProxy>(
757 time_controller_.GetClock(),
758 VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 12:52:48 +0100759 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
760 field_trials_);
Markus Handellee225432021-11-29 12:35:12 +0100761 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_ =
762 CreateBuiltinVideoBitrateAllocatorFactory();
763 VideoStreamEncoderSettings encoder_settings_{
764 VideoEncoder::Capabilities(/*loss_notification=*/false)};
Markus Handell8d87c462021-12-16 11:37:16 +0100765 MockFakeEncoder mock_fake_encoder_{time_controller_.GetClock()};
766 test::VideoEncoderProxyFactory encoder_factory_{&mock_fake_encoder_};
Markus Handellb4e96d42021-11-05 12:00:55 +0100767 NullEncoderSink sink_;
768};
769
770class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
771 public:
772 MOCK_METHOD(void, Initialize, (Callback * callback), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100773 MOCK_METHOD(void,
774 SetZeroHertzModeEnabled,
775 (absl::optional<ZeroHertzModeParams>),
776 (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100777 MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
Markus Handellee225432021-11-29 12:35:12 +0100778 MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override));
779 MOCK_METHOD(void, UpdateFrameRate, (), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100780 MOCK_METHOD(void,
781 UpdateLayerQualityConvergence,
Markus Handell5a77e512022-09-01 12:51:50 +0000782 (size_t spatial_index, bool converged),
Markus Handell8d87c462021-12-16 11:37:16 +0100783 (override));
784 MOCK_METHOD(void,
785 UpdateLayerStatus,
Markus Handell5a77e512022-09-01 12:51:50 +0000786 (size_t spatial_index, bool enabled),
Markus Handell8d87c462021-12-16 11:37:16 +0100787 (override));
Markus Handell818e7fb2021-12-30 13:01:33 +0100788 MOCK_METHOD(void, ProcessKeyFrameRequest, (), (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100789};
790
philipel9b058032020-02-10 11:30:00 +0100791class MockEncoderSelector
792 : public VideoEncoderFactory::EncoderSelectorInterface {
793 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200794 MOCK_METHOD(void,
795 OnCurrentEncoder,
796 (const SdpVideoFormat& format),
797 (override));
798 MOCK_METHOD(absl::optional<SdpVideoFormat>,
799 OnAvailableBitrate,
800 (const DataRate& rate),
801 (override));
philipel6daa3042022-04-11 10:48:28 +0200802 MOCK_METHOD(absl::optional<SdpVideoFormat>,
803 OnResolutionChange,
804 (const RenderResolution& resolution),
805 (override));
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200806 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100807};
808
Markus Handell2e0f4f02021-12-21 19:14:58 +0100809class MockVideoSourceInterface : public rtc::VideoSourceInterface<VideoFrame> {
810 public:
811 MOCK_METHOD(void,
812 AddOrUpdateSink,
813 (rtc::VideoSinkInterface<VideoFrame>*,
814 const rtc::VideoSinkWants&),
815 (override));
816 MOCK_METHOD(void,
817 RemoveSink,
818 (rtc::VideoSinkInterface<VideoFrame>*),
819 (override));
820 MOCK_METHOD(void, RequestRefreshFrame, (), (override));
821};
822
perkj803d97f2016-11-01 11:45:46 -0700823} // namespace
824
mflodmancc3d4422017-08-03 08:27:51 -0700825class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700826 public:
Markus Handell2cfc1af2022-08-19 08:16:48 +0000827 static constexpr TimeDelta kDefaultTimeout = TimeDelta::Seconds(1);
perkj26091b12016-09-01 01:17:40 -0700828
mflodmancc3d4422017-08-03 08:27:51 -0700829 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700830 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700831 codec_width_(320),
832 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200833 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200834 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200835 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700836 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200837 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700838 video_send_config_,
Jonas Oreland8ca06132022-03-14 12:52:48 +0100839 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
840 field_trials_)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200841 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700842
843 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700844 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700845 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200846 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800847 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200848 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200849 video_send_config_.rtp.payload_name = "FAKE";
850 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700851
Per512ecb32016-09-23 15:52:06 +0200852 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200853 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200854 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
855 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
856 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100857 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700858
Niels Möllerf1338562018-04-26 09:51:47 +0200859 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800860 }
861
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100862 void ConfigureEncoder(
863 VideoEncoderConfig video_encoder_config,
864 VideoStreamEncoder::BitrateAllocationCallbackType
865 allocation_callback_type =
866 VideoStreamEncoder::BitrateAllocationCallbackType::
Erik Språnge4589cb2022-04-06 16:44:30 +0200867 kVideoBitrateAllocationWhenScreenSharing,
868 int num_cores = 1) {
mflodmancc3d4422017-08-03 08:27:51 -0700869 if (video_stream_encoder_)
870 video_stream_encoder_->Stop();
Markus Handell9a478b52021-11-18 16:07:01 +0100871
872 auto encoder_queue = GetTaskQueueFactory()->CreateTaskQueue(
873 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
874 TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
875 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter =
876 FrameCadenceAdapterInterface::Create(time_controller_.GetClock(),
Jonas Oreland8ca06132022-03-14 12:52:48 +0100877 encoder_queue_ptr, field_trials_);
Markus Handell9a478b52021-11-18 16:07:01 +0100878 video_stream_encoder_ = std::make_unique<VideoStreamEncoderUnderTest>(
879 &time_controller_, std::move(cadence_adapter), std::move(encoder_queue),
880 stats_proxy_.get(), video_send_config_.encoder_settings,
Erik Språnge4589cb2022-04-06 16:44:30 +0200881 allocation_callback_type, field_trials_, num_cores);
Asa Persson606d3cb2021-10-04 10:07:11 +0200882 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 08:27:51 -0700883 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700884 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 10:07:11 +0200885 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 08:27:51 -0700886 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200887 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700888 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800889 }
890
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100891 void ResetEncoder(const std::string& payload_name,
892 size_t num_streams,
893 size_t num_temporal_layers,
894 unsigned char num_spatial_layers,
895 bool screenshare,
896 VideoStreamEncoder::BitrateAllocationCallbackType
897 allocation_callback_type =
898 VideoStreamEncoder::BitrateAllocationCallbackType::
Erik Språnge4589cb2022-04-06 16:44:30 +0200899 kVideoBitrateAllocationWhenScreenSharing,
900 int num_cores = 1) {
Niels Möller259a4972018-04-05 15:36:51 +0200901 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800902
903 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200904 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
905 num_streams, &video_encoder_config);
906 for (auto& layer : video_encoder_config.simulcast_layers) {
907 layer.num_temporal_layers = num_temporal_layers;
908 layer.max_framerate = kDefaultFramerate;
909 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100910 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 10:07:11 +0200911 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 07:06:52 -0700912 video_encoder_config.content_type =
913 screenshare ? VideoEncoderConfig::ContentType::kScreen
914 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700915 if (payload_name == "VP9") {
916 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
917 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200918 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700919 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200920 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
921 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700922 }
Erik Språnge4589cb2022-04-06 16:44:30 +0200923 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type,
924 num_cores);
perkj26091b12016-09-01 01:17:40 -0700925 }
926
sprang57c2fff2017-01-16 06:24:02 -0800927 VideoFrame CreateFrame(int64_t ntp_time_ms,
928 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200929 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200930 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200931 destruction_event, codec_width_, codec_height_))
932 .set_ntp_time_ms(ntp_time_ms)
933 .set_timestamp_ms(99)
934 .set_rotation(kVideoRotation_0)
935 .build();
perkj26091b12016-09-01 01:17:40 -0700936 }
937
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100938 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
939 rtc::Event* destruction_event,
940 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200941 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200942 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200943 destruction_event, codec_width_, codec_height_))
944 .set_ntp_time_ms(ntp_time_ms)
945 .set_timestamp_ms(99)
946 .set_rotation(kVideoRotation_0)
947 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
948 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100949 }
950
sprang57c2fff2017-01-16 06:24:02 -0800951 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200952 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
953 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200954 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200955 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200956 .set_ntp_time_ms(ntp_time_ms)
957 .set_timestamp_ms(ntp_time_ms)
958 .set_rotation(kVideoRotation_0)
959 .build();
perkj803d97f2016-11-01 11:45:46 -0700960 }
961
Evan Shrubsole895556e2020-10-05 09:15:13 +0200962 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200963 return VideoFrame::Builder()
964 .set_video_frame_buffer(NV12Buffer::Create(width, height))
965 .set_ntp_time_ms(ntp_time_ms)
966 .set_timestamp_ms(ntp_time_ms)
967 .set_rotation(kVideoRotation_0)
968 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200969 }
970
Noah Richards51db4212019-06-12 06:59:12 -0700971 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
972 rtc::Event* destruction_event,
973 int width,
974 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200975 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200976 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200977 destruction_event, width, height))
978 .set_ntp_time_ms(ntp_time_ms)
979 .set_timestamp_ms(99)
980 .set_rotation(kVideoRotation_0)
981 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700982 }
983
Evan Shrubsole895556e2020-10-05 09:15:13 +0200984 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
985 rtc::Event* destruction_event,
986 int width,
987 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200988 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200989 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200990 destruction_event, width, height))
991 .set_ntp_time_ms(ntp_time_ms)
992 .set_timestamp_ms(99)
993 .set_rotation(kVideoRotation_0)
994 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200995 }
996
Noah Richards51db4212019-06-12 06:59:12 -0700997 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
998 rtc::Event* destruction_event) const {
999 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
1000 codec_height_);
1001 }
1002
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001003 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001004 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001005 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001006
1007 video_source_.IncomingCapturedFrame(
1008 CreateFrame(1, codec_width_, codec_height_));
1009 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +02001010 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001011 }
1012
sprang4847ae62017-06-27 07:06:52 -07001013 void WaitForEncodedFrame(int64_t expected_ntp_time) {
1014 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001015 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001016 }
1017
Markus Handell2cfc1af2022-08-19 08:16:48 +00001018 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, TimeDelta timeout) {
1019 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001020 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001021 return ok;
1022 }
1023
1024 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
1025 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001026 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001027 }
1028
1029 void ExpectDroppedFrame() {
1030 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001031 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001032 }
1033
Markus Handell2cfc1af2022-08-19 08:16:48 +00001034 bool WaitForFrame(TimeDelta timeout) {
1035 bool ok = sink_.WaitForFrame(timeout);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001036 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001037 return ok;
1038 }
1039
perkj26091b12016-09-01 01:17:40 -07001040 class TestEncoder : public test::FakeEncoder {
1041 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001042 explicit TestEncoder(TimeController* time_controller)
1043 : FakeEncoder(time_controller->GetClock()),
1044 time_controller_(time_controller) {
1045 RTC_DCHECK(time_controller_);
1046 }
perkj26091b12016-09-01 01:17:40 -07001047
Erik Språngaed30702018-11-05 12:57:17 +01001048 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +02001049 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +02001050 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +01001051 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001052 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +01001053 info.scaling_settings = VideoEncoder::ScalingSettings(
1054 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001055 }
1056 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001057 for (int i = 0; i < kMaxSpatialLayers; ++i) {
1058 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001059 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001060 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001061 for (int tid = 0; tid < num_layers; ++tid)
1062 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001063 }
1064 }
Erik Språngaed30702018-11-05 12:57:17 +01001065 }
Sergey Silkin6456e352019-07-08 17:56:40 +02001066
1067 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001068 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001069 info.apply_alignment_to_all_simulcast_layers =
1070 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001071 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001072 if (is_qp_trusted_.has_value()) {
1073 info.is_qp_trusted = is_qp_trusted_;
1074 }
Erik Språngaed30702018-11-05 12:57:17 +01001075 return info;
kthelgason876222f2016-11-29 01:44:11 -08001076 }
1077
Erik Språngb7cb7b52019-02-26 15:52:33 +01001078 int32_t RegisterEncodeCompleteCallback(
1079 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +02001080 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001081 encoded_image_callback_ = callback;
1082 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
1083 }
1084
perkjfa10b552016-10-02 23:45:26 -07001085 void ContinueEncode() { continue_encode_event_.Set(); }
1086
1087 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
1088 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +02001089 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -07001090 EXPECT_EQ(timestamp_, timestamp);
1091 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
1092 }
1093
kthelgason2fc52542017-03-03 00:24:41 -08001094 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +02001095 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -08001096 quality_scaling_ = b;
1097 }
kthelgasonad9010c2017-02-14 00:46:51 -08001098
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001099 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +02001100 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001101 requested_resolution_alignment_ = requested_resolution_alignment;
1102 }
1103
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001104 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
1105 MutexLock lock(&local_mutex_);
1106 apply_alignment_to_all_simulcast_layers_ = b;
1107 }
1108
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001109 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +02001110 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001111 is_hardware_accelerated_ = is_hardware_accelerated;
1112 }
1113
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001114 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
1115 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +02001116 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001117 temporal_layers_supported_[spatial_idx] = supported;
1118 }
1119
Sergey Silkin6456e352019-07-08 17:56:40 +02001120 void SetResolutionBitrateLimits(
1121 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +02001122 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +02001123 resolution_bitrate_limits_ = thresholds;
1124 }
1125
sprangfe627f32017-03-29 08:24:59 -07001126 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +02001127 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -07001128 force_init_encode_failed_ = force_failure;
1129 }
1130
Niels Möller6bb5ab92019-01-11 11:11:10 +01001131 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +02001132 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001133 rate_factor_ = rate_factor;
1134 }
1135
Erik Språngd7329ca2019-02-21 21:19:53 +01001136 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +02001137 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001138 return last_framerate_;
1139 }
1140
Erik Språngd7329ca2019-02-21 21:19:53 +01001141 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +02001142 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001143 return last_update_rect_;
1144 }
1145
Niels Möller87e2d782019-03-07 10:18:23 +01001146 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +02001147 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001148 return last_frame_types_;
1149 }
1150
1151 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +01001152 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +01001153 keyframe ? VideoFrameType::kVideoFrameKey
1154 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01001155 {
Markus Handella3765182020-07-08 13:13:32 +02001156 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001157 last_frame_types_ = frame_type;
1158 }
Niels Möllerb859b322019-03-07 12:40:01 +01001159 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +01001160 }
1161
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001162 void InjectEncodedImage(const EncodedImage& image,
1163 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +02001164 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001165 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001166 }
1167
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001168 void SetEncodedImageData(
1169 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +02001170 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001171 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001172 }
1173
Erik Språngd7329ca2019-02-21 21:19:53 +01001174 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +02001175 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001176 expect_null_frame_ = true;
1177 }
1178
Erik Språng5056af02019-09-02 15:53:11 +02001179 absl::optional<VideoEncoder::RateControlParameters>
1180 GetAndResetLastRateControlSettings() {
1181 auto settings = last_rate_control_settings_;
1182 last_rate_control_settings_.reset();
1183 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +01001184 }
1185
Henrik Boström56db9ff2021-03-24 09:06:45 +01001186 int GetLastInputWidth() const {
1187 MutexLock lock(&local_mutex_);
1188 return last_input_width_;
1189 }
1190
1191 int GetLastInputHeight() const {
1192 MutexLock lock(&local_mutex_);
1193 return last_input_height_;
1194 }
1195
Evan Shrubsole895556e2020-10-05 09:15:13 +02001196 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1197 MutexLock lock(&local_mutex_);
1198 return last_input_pixel_format_;
1199 }
1200
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001201 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001202 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001203 return num_set_rates_;
1204 }
1205
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001206 void SetPreferredPixelFormats(
1207 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1208 pixel_formats) {
1209 MutexLock lock(&local_mutex_);
1210 preferred_pixel_formats_ = std::move(pixel_formats);
1211 }
1212
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001213 void SetIsQpTrusted(absl::optional<bool> trusted) {
1214 MutexLock lock(&local_mutex_);
1215 is_qp_trusted_ = trusted;
1216 }
1217
Erik Språnge4589cb2022-04-06 16:44:30 +02001218 VideoCodecComplexity LastEncoderComplexity() {
1219 MutexLock lock(&local_mutex_);
1220 return last_encoder_complexity_;
1221 }
1222
perkjfa10b552016-10-02 23:45:26 -07001223 private:
perkj26091b12016-09-01 01:17:40 -07001224 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001225 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001226 {
Markus Handella3765182020-07-08 13:13:32 +02001227 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001228 if (expect_null_frame_) {
1229 EXPECT_EQ(input_image.timestamp(), 0u);
1230 EXPECT_EQ(input_image.width(), 1);
1231 last_frame_types_ = *frame_types;
1232 expect_null_frame_ = false;
1233 } else {
1234 EXPECT_GT(input_image.timestamp(), timestamp_);
1235 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1236 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1237 }
perkj26091b12016-09-01 01:17:40 -07001238
1239 timestamp_ = input_image.timestamp();
1240 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001241 last_input_width_ = input_image.width();
1242 last_input_height_ = input_image.height();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001243 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001244 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001245 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001246 }
Niels Möllerb859b322019-03-07 12:40:01 +01001247 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001248 return result;
1249 }
1250
Niels Möller08ae7ce2020-09-23 15:58:12 +02001251 CodecSpecificInfo EncodeHook(
1252 EncodedImage& encoded_image,
1253 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001254 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001255 {
1256 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001257 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001258 }
1259 MutexLock lock(&local_mutex_);
1260 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001261 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001262 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001263 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001264 }
1265
sprangfe627f32017-03-29 08:24:59 -07001266 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001267 const Settings& settings) override {
1268 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001269
Markus Handella3765182020-07-08 13:13:32 +02001270 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001271 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001272
Erik Språng82fad3d2018-03-21 09:57:23 +01001273 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001274 // Simulate setting up temporal layers, in order to validate the life
1275 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001276 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001277 frame_buffer_controller_ =
1278 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001279 }
Erik Språnge4589cb2022-04-06 16:44:30 +02001280
1281 last_encoder_complexity_ = config->GetVideoEncoderComplexity();
1282
Erik Språngb7cb7b52019-02-26 15:52:33 +01001283 if (force_init_encode_failed_) {
1284 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001285 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001286 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001287
Erik Språngb7cb7b52019-02-26 15:52:33 +01001288 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001289 return res;
1290 }
1291
Erik Språngb7cb7b52019-02-26 15:52:33 +01001292 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001293 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001294 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1295 initialized_ = EncoderState::kUninitialized;
1296 return FakeEncoder::Release();
1297 }
1298
Erik Språng16cb8f52019-04-12 13:59:09 +02001299 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001300 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001301 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001302 VideoBitrateAllocation adjusted_rate_allocation;
1303 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1304 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001305 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001306 adjusted_rate_allocation.SetBitrate(
1307 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001308 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001309 rate_factor_));
1310 }
1311 }
1312 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001313 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001314 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001315 RateControlParameters adjusted_paramters = parameters;
1316 adjusted_paramters.bitrate = adjusted_rate_allocation;
1317 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001318 }
1319
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001320 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001321 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001322 enum class EncoderState {
1323 kUninitialized,
1324 kInitializationFailed,
1325 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001326 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
perkj26091b12016-09-01 01:17:40 -07001327 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001328 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1329 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1330 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1331 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1332 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1333 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001334 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1335 false;
Markus Handella3765182020-07-08 13:13:32 +02001336 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001337 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1338 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001339 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001340 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001341 absl::optional<bool>
1342 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001343 local_mutex_);
1344 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1345 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1346 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001347 absl::optional<VideoEncoder::RateControlParameters>
1348 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001349 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1350 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001351 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001352 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001353 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1354 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001355 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 17:56:40 +02001356 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001357 RTC_GUARDED_BY(local_mutex_);
1358 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001359 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1360 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001361 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1362 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001363 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
Erik Språnge4589cb2022-04-06 16:44:30 +02001364 VideoCodecComplexity last_encoder_complexity_ RTC_GUARDED_BY(local_mutex_){
1365 VideoCodecComplexity::kComplexityNormal};
perkj26091b12016-09-01 01:17:40 -07001366 };
1367
mflodmancc3d4422017-08-03 08:27:51 -07001368 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001369 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001370 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1371 : time_controller_(time_controller), test_encoder_(test_encoder) {
1372 RTC_DCHECK(time_controller_);
1373 }
perkj26091b12016-09-01 01:17:40 -07001374
perkj26091b12016-09-01 01:17:40 -07001375 void WaitForEncodedFrame(int64_t expected_ntp_time) {
Markus Handell2cfc1af2022-08-19 08:16:48 +00001376 EXPECT_TRUE(TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeout));
sprang4847ae62017-06-27 07:06:52 -07001377 }
1378
1379 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
Markus Handell2cfc1af2022-08-19 08:16:48 +00001380 TimeDelta timeout) {
perkj26091b12016-09-01 01:17:40 -07001381 uint32_t timestamp = 0;
Markus Handell2cfc1af2022-08-19 08:16:48 +00001382 if (!WaitForFrame(timeout))
sprang4847ae62017-06-27 07:06:52 -07001383 return false;
perkj26091b12016-09-01 01:17:40 -07001384 {
Markus Handella3765182020-07-08 13:13:32 +02001385 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001386 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001387 }
1388 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001389 return true;
perkj26091b12016-09-01 01:17:40 -07001390 }
1391
sprangb1ca0732017-02-01 08:38:12 -08001392 void WaitForEncodedFrame(uint32_t expected_width,
1393 uint32_t expected_height) {
Markus Handell2cfc1af2022-08-19 08:16:48 +00001394 EXPECT_TRUE(WaitForFrame(kDefaultTimeout));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001395 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001396 }
1397
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001398 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001399 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001400 uint32_t width = 0;
1401 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001402 {
Markus Handella3765182020-07-08 13:13:32 +02001403 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001404 width = last_width_;
1405 height = last_height_;
1406 }
1407 EXPECT_EQ(expected_height, height);
1408 EXPECT_EQ(expected_width, width);
1409 }
1410
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001411 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1412 VideoRotation rotation;
1413 {
Markus Handella3765182020-07-08 13:13:32 +02001414 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001415 rotation = last_rotation_;
1416 }
1417 EXPECT_EQ(expected_rotation, rotation);
1418 }
1419
Markus Handell2cfc1af2022-08-19 08:16:48 +00001420 void ExpectDroppedFrame() {
1421 EXPECT_FALSE(WaitForFrame(TimeDelta::Millis(100)));
1422 }
kthelgason2bc68642017-02-07 07:02:22 -08001423
Markus Handell2cfc1af2022-08-19 08:16:48 +00001424 bool WaitForFrame(TimeDelta timeout) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001425 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +01001426 time_controller_->AdvanceTime(TimeDelta::Zero());
Markus Handell2cfc1af2022-08-19 08:16:48 +00001427 bool ret = encoded_frame_event_.Wait(timeout);
Markus Handell28c71802021-11-08 10:11:55 +01001428 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001429 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001430 }
1431
perkj26091b12016-09-01 01:17:40 -07001432 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001433 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001434 expect_frames_ = false;
1435 }
1436
asaperssonfab67072017-04-04 05:51:49 -07001437 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001438 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001439 return number_of_reconfigurations_;
1440 }
1441
asaperssonfab67072017-04-04 05:51:49 -07001442 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001443 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001444 return min_transmit_bitrate_bps_;
1445 }
1446
Erik Språngd7329ca2019-02-21 21:19:53 +01001447 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001448 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001449 num_expected_layers_ = num_layers;
1450 }
1451
Erik Språngb7cb7b52019-02-26 15:52:33 +01001452 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001453 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001454 return last_capture_time_ms_;
1455 }
1456
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001457 const EncodedImage& GetLastEncodedImage() {
1458 MutexLock lock(&mutex_);
1459 return last_encoded_image_;
1460 }
1461
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001462 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001463 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001464 return std::move(last_encoded_image_data_);
1465 }
1466
Per Kjellanderdcef6412020-10-07 15:09:05 +02001467 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1468 MutexLock lock(&mutex_);
1469 return last_bitrate_allocation_;
1470 }
1471
1472 int number_of_bitrate_allocations() const {
1473 MutexLock lock(&mutex_);
1474 return number_of_bitrate_allocations_;
1475 }
1476
Per Kjellandera9434842020-10-15 17:53:22 +02001477 VideoLayersAllocation GetLastVideoLayersAllocation() {
1478 MutexLock lock(&mutex_);
1479 return last_layers_allocation_;
1480 }
1481
1482 int number_of_layers_allocations() const {
1483 MutexLock lock(&mutex_);
1484 return number_of_layers_allocations_;
1485 }
1486
perkj26091b12016-09-01 01:17:40 -07001487 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001488 Result OnEncodedImage(
1489 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001490 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001491 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001492 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001493 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001494 last_encoded_image_data_ = std::vector<uint8_t>(
1495 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001496 uint32_t timestamp = encoded_image.Timestamp();
1497 if (last_timestamp_ != timestamp) {
1498 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001499 last_width_ = encoded_image._encodedWidth;
1500 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001501 } else {
1502 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001503 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1504 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001505 }
1506 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001507 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001508 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001509 if (num_received_layers_ == num_expected_layers_) {
1510 encoded_frame_event_.Set();
1511 }
sprangb1ca0732017-02-01 08:38:12 -08001512 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001513 }
1514
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001515 void OnEncoderConfigurationChanged(
1516 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001517 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001518 VideoEncoderConfig::ContentType content_type,
1519 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001520 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001521 ++number_of_reconfigurations_;
1522 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1523 }
1524
Per Kjellanderdcef6412020-10-07 15:09:05 +02001525 void OnBitrateAllocationUpdated(
1526 const VideoBitrateAllocation& allocation) override {
1527 MutexLock lock(&mutex_);
1528 ++number_of_bitrate_allocations_;
1529 last_bitrate_allocation_ = allocation;
1530 }
1531
Per Kjellandera9434842020-10-15 17:53:22 +02001532 void OnVideoLayersAllocationUpdated(
1533 VideoLayersAllocation allocation) override {
1534 MutexLock lock(&mutex_);
1535 ++number_of_layers_allocations_;
1536 last_layers_allocation_ = allocation;
1537 rtc::StringBuilder log;
1538 for (const auto& layer : allocation.active_spatial_layers) {
1539 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1540 << "[";
1541 for (const auto target_bitrate :
1542 layer.target_bitrate_per_temporal_layer) {
1543 log << target_bitrate.kbps() << ",";
1544 }
1545 log << "]";
1546 }
Harald Alvestrand97597c02021-11-04 12:01:23 +00001547 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 17:53:22 +02001548 }
1549
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001550 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001551 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001552 TestEncoder* test_encoder_;
1553 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001554 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001555 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001556 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001557 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001558 uint32_t last_height_ = 0;
1559 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001560 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001561 size_t num_expected_layers_ = 1;
1562 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001563 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001564 int number_of_reconfigurations_ = 0;
1565 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001566 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1567 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001568 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1569 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001570 };
1571
Sergey Silkin5ee69672019-07-02 14:18:34 +02001572 class VideoBitrateAllocatorProxyFactory
1573 : public VideoBitrateAllocatorFactory {
1574 public:
1575 VideoBitrateAllocatorProxyFactory()
1576 : bitrate_allocator_factory_(
1577 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1578
1579 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1580 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001581 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001582 codec_config_ = codec;
1583 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1584 }
1585
1586 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001587 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001588 return codec_config_;
1589 }
1590
1591 private:
1592 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1593
Markus Handella3765182020-07-08 13:13:32 +02001594 mutable Mutex mutex_;
1595 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001596 };
1597
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001598 Clock* clock() { return time_controller_.GetClock(); }
1599 void AdvanceTime(TimeDelta duration) {
1600 time_controller_.AdvanceTime(duration);
1601 }
1602
1603 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1604
1605 protected:
1606 virtual TaskQueueFactory* GetTaskQueueFactory() {
1607 return time_controller_.GetTaskQueueFactory();
1608 }
1609
Jonas Orelandc7f691a2022-03-09 15:12:07 +01001610 test::ScopedKeyValueConfig field_trials_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001611 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001612 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001613 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001614 int codec_width_;
1615 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001616 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001617 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001618 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001619 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001620 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001621 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001622 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001623 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001624};
1625
mflodmancc3d4422017-08-03 08:27:51 -07001626TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001627 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001628 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001629 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001630 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001631 WaitForEncodedFrame(1);
Markus Handell2cfc1af2022-08-19 08:16:48 +00001632 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
mflodmancc3d4422017-08-03 08:27:51 -07001633 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001634}
1635
mflodmancc3d4422017-08-03 08:27:51 -07001636TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001637 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001638 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001639 // The encoder will cache up to one frame for a short duration. Adding two
1640 // frames means that the first frame will be dropped and the second frame will
1641 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001642 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001643 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001644 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Markus Handell28c71802021-11-08 10:11:55 +01001645 AdvanceTime(TimeDelta::Zero());
Markus Handell2cfc1af2022-08-19 08:16:48 +00001646 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
perkj26091b12016-09-01 01:17:40 -07001647
Henrik Boström381d1092020-05-12 18:49:07 +02001648 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001649 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001650
Sebastian Janssona3177052018-04-10 13:05:49 +02001651 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001652 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001653 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1654
1655 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001656 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001657}
1658
mflodmancc3d4422017-08-03 08:27:51 -07001659TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001660 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001661 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001662 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001663 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001664
Henrik Boström381d1092020-05-12 18:49:07 +02001665 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001666 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1667
Sebastian Janssona3177052018-04-10 13:05:49 +02001668 // The encoder will cache up to one frame for a short duration. Adding two
1669 // frames means that the first frame will be dropped and the second frame will
1670 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001671 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001672 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001673
Henrik Boström381d1092020-05-12 18:49:07 +02001674 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001675 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001676 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001677 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1678 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001679 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001680}
1681
mflodmancc3d4422017-08-03 08:27:51 -07001682TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001683 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001684 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001685 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001686 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001687
1688 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001689 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001690
perkja49cbd32016-09-16 07:53:41 -07001691 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001692 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001693 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001694}
1695
mflodmancc3d4422017-08-03 08:27:51 -07001696TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001697 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001698 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001699
perkja49cbd32016-09-16 07:53:41 -07001700 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001701 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001702
mflodmancc3d4422017-08-03 08:27:51 -07001703 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001704 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001705 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001706 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
Markus Handell2cfc1af2022-08-19 08:16:48 +00001707 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
perkj26091b12016-09-01 01:17:40 -07001708}
1709
Markus Handell9a478b52021-11-18 16:07:01 +01001710TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
1711 test::FrameForwarder source;
1712 video_stream_encoder_->SetSource(&source,
1713 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02001714 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001715 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001716
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001717 int dropped_count = 0;
1718 stats_proxy_->SetDroppedFrameCallback(
1719 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1720 ++dropped_count;
1721 });
1722
Markus Handell9a478b52021-11-18 16:07:01 +01001723 source.IncomingCapturedFrame(CreateFrame(1, nullptr));
1724 source.IncomingCapturedFrame(CreateFrame(2, nullptr));
1725 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001726 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001727 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001728}
1729
Henrik Boström56db9ff2021-03-24 09:06:45 +01001730TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001731 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001732 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001733
1734 rtc::Event frame_destroyed_event;
1735 video_source_.IncomingCapturedFrame(
1736 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001737 WaitForEncodedFrame(1);
1738 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1739 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001740 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1741 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001742 video_stream_encoder_->Stop();
1743}
1744
Henrik Boström56db9ff2021-03-24 09:06:45 +01001745TEST_F(VideoStreamEncoderTest,
1746 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001747 // Use the cropping factory.
1748 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001749 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001750 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1751 kMaxPayloadLength);
1752 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1753
1754 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001755 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001756 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001757 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1758 WaitForEncodedFrame(1);
1759 // The encoder will have been configured once.
1760 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001761 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1762 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 06:59:12 -07001763
1764 // Now send in a fake frame that needs to be cropped as the width/height
1765 // aren't divisible by 4 (see CreateEncoderStreams above).
1766 rtc::Event frame_destroyed_event;
1767 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1768 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001769 WaitForEncodedFrame(2);
1770 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1771 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001772 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1773 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001774 video_stream_encoder_->Stop();
1775}
1776
Evan Shrubsole895556e2020-10-05 09:15:13 +02001777TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1778 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001779 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001780
1781 video_source_.IncomingCapturedFrame(
1782 CreateNV12Frame(1, codec_width_, codec_height_));
1783 WaitForEncodedFrame(1);
1784 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1785 fake_encoder_.GetLastInputPixelFormat());
1786 video_stream_encoder_->Stop();
1787}
1788
Henrik Boström56db9ff2021-03-24 09:06:45 +01001789TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001790 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001791 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001792
1793 fake_encoder_.SetPreferredPixelFormats({});
1794
1795 rtc::Event frame_destroyed_event;
1796 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1797 1, &frame_destroyed_event, codec_width_, codec_height_));
1798 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001799 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001800 fake_encoder_.GetLastInputPixelFormat());
1801 video_stream_encoder_->Stop();
1802}
1803
Henrik Boström56db9ff2021-03-24 09:06:45 +01001804TEST_F(VideoStreamEncoderTest,
1805 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001806 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001807 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001808
1809 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1810
1811 rtc::Event frame_destroyed_event;
1812 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1813 1, &frame_destroyed_event, codec_width_, codec_height_));
1814 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001815 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001816 fake_encoder_.GetLastInputPixelFormat());
1817 video_stream_encoder_->Stop();
1818}
1819
Henrik Boström56db9ff2021-03-24 09:06:45 +01001820TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001821 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001822 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001823
1824 // Fake NV12 native frame does not allow mapping to I444.
1825 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1826
1827 rtc::Event frame_destroyed_event;
1828 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1829 1, &frame_destroyed_event, codec_width_, codec_height_));
1830 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001831 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001832 fake_encoder_.GetLastInputPixelFormat());
1833 video_stream_encoder_->Stop();
1834}
1835
Henrik Boström56db9ff2021-03-24 09:06:45 +01001836TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001837 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001838 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001839
1840 rtc::Event frame_destroyed_event;
1841 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1842 1, &frame_destroyed_event, codec_width_, codec_height_));
1843 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001844 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001845 fake_encoder_.GetLastInputPixelFormat());
1846 video_stream_encoder_->Stop();
1847}
1848
Ying Wang9b881ab2020-02-07 14:29:32 +01001849TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001850 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001851 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001852 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1853 WaitForEncodedFrame(1);
1854
Henrik Boström381d1092020-05-12 18:49:07 +02001855 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001856 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001857 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1858 // frames. Adding two frames means that the first frame will be dropped and
1859 // the second frame will be sent to the encoder.
1860 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1861 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1862 WaitForEncodedFrame(3);
1863 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1864 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1865 WaitForEncodedFrame(5);
1866 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1867 video_stream_encoder_->Stop();
1868}
1869
mflodmancc3d4422017-08-03 08:27:51 -07001870TEST_F(VideoStreamEncoderTest,
1871 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001872 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001873 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001874 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001875
1876 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001877 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001878 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001879 // The encoder will have been configured once when the first frame is
1880 // received.
1881 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001882
1883 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001884 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001885 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001886 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001887 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001888
1889 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001890 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001891 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001892 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001893 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001894
mflodmancc3d4422017-08-03 08:27:51 -07001895 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001896}
1897
mflodmancc3d4422017-08-03 08:27:51 -07001898TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001899 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001900 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001901
1902 // Capture a frame and wait for it to synchronize with the encoder thread.
1903 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001904 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001905 // The encoder will have been configured once.
1906 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001907 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1908 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-02 23:45:26 -07001909
1910 codec_width_ *= 2;
1911 codec_height_ *= 2;
1912 // Capture a frame with a higher resolution and wait for it to synchronize
1913 // with the encoder thread.
1914 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001915 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 10:07:11 +02001916 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1917 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 21:37:57 +01001918 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001919
mflodmancc3d4422017-08-03 08:27:51 -07001920 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001921}
1922
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001923TEST_F(VideoStreamEncoderTest,
1924 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001925 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001926 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001927
1928 // Capture a frame and wait for it to synchronize with the encoder thread.
1929 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1930 WaitForEncodedFrame(1);
1931
1932 VideoEncoderConfig video_encoder_config;
1933 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1934 // Changing the max payload data length recreates encoder.
1935 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1936 kMaxPayloadLength / 2);
1937
1938 // Capture a frame and wait for it to synchronize with the encoder thread.
1939 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1940 WaitForEncodedFrame(2);
1941 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1942
1943 video_stream_encoder_->Stop();
1944}
1945
Sergey Silkin5ee69672019-07-02 14:18:34 +02001946TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001947 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001948 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001949
1950 VideoEncoderConfig video_encoder_config;
1951 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02001952 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1953 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001954 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1955 kMaxPayloadLength);
1956
1957 // Capture a frame and wait for it to synchronize with the encoder thread.
1958 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1959 WaitForEncodedFrame(1);
1960 // The encoder will have been configured once when the first frame is
1961 // received.
1962 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001963 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001964 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001965 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001966 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1967
Sergey Silkin6456e352019-07-08 17:56:40 +02001968 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1969 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 10:07:11 +02001970 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1971 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001972 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1973 kMaxPayloadLength);
1974
1975 // Capture a frame and wait for it to synchronize with the encoder thread.
1976 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1977 WaitForEncodedFrame(2);
1978 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1979 // Bitrate limits have changed - rate allocator should be reconfigured,
1980 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 10:07:11 +02001981 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001982 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001983 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001984 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001985 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001986
1987 video_stream_encoder_->Stop();
1988}
1989
Sergey Silkin6456e352019-07-08 17:56:40 +02001990TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001991 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001992 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001993 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001994
Sergey Silkincd02eba2020-01-20 14:48:40 +01001995 const uint32_t kMinEncBitrateKbps = 100;
1996 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001997 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001998 /*frame_size_pixels=*/codec_width_ * codec_height_,
1999 /*min_start_bitrate_bps=*/0,
2000 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
2001 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002002 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2003
Sergey Silkincd02eba2020-01-20 14:48:40 +01002004 VideoEncoderConfig video_encoder_config;
2005 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2006 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
2007 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2008 (kMinEncBitrateKbps + 1) * 1000;
2009 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2010 kMaxPayloadLength);
2011
2012 // When both encoder and app provide bitrate limits, the intersection of
2013 // provided sets should be used.
2014 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2015 WaitForEncodedFrame(1);
2016 EXPECT_EQ(kMaxEncBitrateKbps,
2017 bitrate_allocator_factory_.codec_config().maxBitrate);
2018 EXPECT_EQ(kMinEncBitrateKbps + 1,
2019 bitrate_allocator_factory_.codec_config().minBitrate);
2020
2021 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
2022 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2023 (kMinEncBitrateKbps - 1) * 1000;
2024 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2025 kMaxPayloadLength);
2026 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002027 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002028 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002029 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002030 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002031 bitrate_allocator_factory_.codec_config().minBitrate);
2032
Sergey Silkincd02eba2020-01-20 14:48:40 +01002033 video_stream_encoder_->Stop();
2034}
2035
2036TEST_F(VideoStreamEncoderTest,
2037 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02002038 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002039 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002040
2041 const uint32_t kMinAppBitrateKbps = 100;
2042 const uint32_t kMaxAppBitrateKbps = 200;
2043 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
2044 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
2045 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2046 /*frame_size_pixels=*/codec_width_ * codec_height_,
2047 /*min_start_bitrate_bps=*/0,
2048 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
2049 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
2050 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2051
2052 VideoEncoderConfig video_encoder_config;
2053 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2054 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
2055 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2056 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002057 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2058 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002059
Sergey Silkincd02eba2020-01-20 14:48:40 +01002060 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2061 WaitForEncodedFrame(1);
2062 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002063 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002064 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002065 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02002066
2067 video_stream_encoder_->Stop();
2068}
2069
2070TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002071 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02002072 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002073 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02002074
2075 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002076 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002077 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002078 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002079 fake_encoder_.SetResolutionBitrateLimits(
2080 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
2081
2082 VideoEncoderConfig video_encoder_config;
2083 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2084 video_encoder_config.max_bitrate_bps = 0;
2085 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2086 kMaxPayloadLength);
2087
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002088 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002089 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2090 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002091 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2092 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002093 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2094 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2095
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002096 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002097 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2098 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002099 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2100 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002101 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2102 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2103
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002104 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02002105 // encoder for 360p should be used.
2106 video_source_.IncomingCapturedFrame(
2107 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2108 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002109 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2110 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002111 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2112 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2113
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002114 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02002115 // ignored.
2116 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2117 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002118 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2119 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002120 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2121 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002122 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2123 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002124 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2125 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2126
2127 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2128 // for 270p should be used.
2129 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2130 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002131 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2132 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002133 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2134 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2135
2136 video_stream_encoder_->Stop();
2137}
2138
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002139TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002140 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002141 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002142
2143 VideoEncoderConfig video_encoder_config;
2144 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2145 video_encoder_config.max_bitrate_bps = 0;
2146 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2147 kMaxPayloadLength);
2148
2149 // Encode 720p frame to get the default encoder target bitrate.
2150 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2151 WaitForEncodedFrame(1);
2152 const uint32_t kDefaultTargetBitrateFor720pKbps =
2153 bitrate_allocator_factory_.codec_config()
2154 .simulcastStream[0]
2155 .targetBitrate;
2156
2157 // Set the max recommended encoder bitrate to something lower than the default
2158 // target bitrate.
2159 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2160 1280 * 720, 10 * 1000, 10 * 1000,
2161 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2162 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2163
2164 // Change resolution to trigger encoder reinitialization.
2165 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2166 WaitForEncodedFrame(2);
2167 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2168 WaitForEncodedFrame(3);
2169
2170 // Ensure the target bitrate is capped by the max bitrate.
2171 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2172 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2173 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2174 .simulcastStream[0]
2175 .targetBitrate *
2176 1000,
2177 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2178
2179 video_stream_encoder_->Stop();
2180}
2181
Åsa Perssona7e34d32021-01-20 15:36:13 +01002182TEST_F(VideoStreamEncoderTest,
2183 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2184 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2185 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2186 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2187 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2188 fake_encoder_.SetResolutionBitrateLimits(
2189 {kEncoderLimits270p, kEncoderLimits360p});
2190
2191 // Two streams, highest stream active.
2192 VideoEncoderConfig config;
2193 const int kNumStreams = 2;
2194 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2195 config.max_bitrate_bps = 0;
2196 config.simulcast_layers[0].active = false;
2197 config.simulcast_layers[1].active = true;
2198 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002199 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002200 "VP8", /*max qp*/ 56, /*screencast*/ false,
2201 /*screenshare enabled*/ false);
2202 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2203
2204 // The encoder bitrate limits for 270p should be used.
2205 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
Erik Språngf449af82022-08-08 17:54:55 +02002206 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002207 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002208 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002209 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002210 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002211 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002212
2213 // The encoder bitrate limits for 360p should be used.
2214 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
Erik Språngf449af82022-08-08 17:54:55 +02002215 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002216 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002217 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002218 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002219 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002220
2221 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2222 video_source_.IncomingCapturedFrame(
2223 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
Erik Språngf449af82022-08-08 17:54:55 +02002224 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002225 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002226 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002227 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002228 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002229
2230 // Resolution higher than 360p. Encoder limits should be ignored.
2231 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02002232 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002233 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002234 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002235 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002236 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002237 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002238 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002239 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002240 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002241
2242 // Resolution lower than 270p. The encoder limits for 270p should be used.
2243 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
Erik Språngf449af82022-08-08 17:54:55 +02002244 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002245 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002246 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002247 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002248 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002249
2250 video_stream_encoder_->Stop();
2251}
2252
2253TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002254 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2255 // Two streams, highest stream active.
2256 VideoEncoderConfig config;
2257 const int kNumStreams = 2;
2258 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2259 config.max_bitrate_bps = 0;
2260 config.simulcast_layers[0].active = false;
2261 config.simulcast_layers[1].active = true;
2262 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002263 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002264 "VP8", /*max qp*/ 56, /*screencast*/ false,
2265 /*screenshare enabled*/ false);
2266 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2267
2268 // Default bitrate limits for 270p should be used.
2269 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2270 kDefaultLimits270p =
2271 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002272 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002273 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
Erik Språngf449af82022-08-08 17:54:55 +02002274 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002275 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 10:39:51 +01002276 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002277 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002278 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002279 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002280
2281 // Default bitrate limits for 360p should be used.
2282 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2283 kDefaultLimits360p =
2284 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002285 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002286 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
Erik Språngf449af82022-08-08 17:54:55 +02002287 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Persson258e9892021-02-25 10:39:51 +01002288 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002289 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002290 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002291 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002292
2293 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2294 video_source_.IncomingCapturedFrame(
2295 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
Erik Språngf449af82022-08-08 17:54:55 +02002296 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Persson258e9892021-02-25 10:39:51 +01002297 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002298 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002299 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002300 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002301
2302 // Default bitrate limits for 540p should be used.
2303 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2304 kDefaultLimits540p =
2305 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002306 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002307 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02002308 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Persson258e9892021-02-25 10:39:51 +01002309 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002310 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002311 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002312 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002313
2314 video_stream_encoder_->Stop();
2315}
2316
2317TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002318 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2319 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2320 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2321 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2322 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2323 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2324 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2325 fake_encoder_.SetResolutionBitrateLimits(
2326 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2327
2328 // Three streams, middle stream active.
2329 VideoEncoderConfig config;
2330 const int kNumStreams = 3;
2331 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2332 config.simulcast_layers[0].active = false;
2333 config.simulcast_layers[1].active = true;
2334 config.simulcast_layers[2].active = false;
2335 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002336 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002337 "VP8", /*max qp*/ 56, /*screencast*/ false,
2338 /*screenshare enabled*/ false);
2339 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2340
2341 // The encoder bitrate limits for 360p should be used.
2342 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02002343 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002344 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002345 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002346 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002347 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002348 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002349
2350 // The encoder bitrate limits for 270p should be used.
2351 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02002352 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002353 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002354 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002355 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002356 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002357
2358 video_stream_encoder_->Stop();
2359}
2360
2361TEST_F(VideoStreamEncoderTest,
2362 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2363 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2364 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2365 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2366 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2367 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2368 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2369 fake_encoder_.SetResolutionBitrateLimits(
2370 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2371
2372 // Three streams, lowest stream active.
2373 VideoEncoderConfig config;
2374 const int kNumStreams = 3;
2375 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2376 config.simulcast_layers[0].active = true;
2377 config.simulcast_layers[1].active = false;
2378 config.simulcast_layers[2].active = false;
2379 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002380 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002381 "VP8", /*max qp*/ 56, /*screencast*/ false,
2382 /*screenshare enabled*/ false);
2383 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2384
2385 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2386 // on lowest stream, limits for 270p should not be used
2387 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02002388 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002389 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002390 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002391 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002392 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002393 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002394
2395 video_stream_encoder_->Stop();
2396}
2397
2398TEST_F(VideoStreamEncoderTest,
2399 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2400 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2401 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2402 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2403 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2404 fake_encoder_.SetResolutionBitrateLimits(
2405 {kEncoderLimits270p, kEncoderLimits360p});
2406 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2407
2408 // Two streams, highest stream active.
2409 VideoEncoderConfig config;
2410 const int kNumStreams = 2;
2411 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2412 config.simulcast_layers[0].active = false;
2413 config.simulcast_layers[1].active = true;
2414 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2415 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002416 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002417 "VP8", /*max qp*/ 56, /*screencast*/ false,
2418 /*screenshare enabled*/ false);
2419 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2420
2421 // The encoder bitrate limits for 270p should be used.
2422 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
Erik Språngf449af82022-08-08 17:54:55 +02002423 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002424 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002425 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002426 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002427 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002428 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002429
2430 // The max configured bitrate is less than the encoder limit for 360p.
2431 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
Erik Språngf449af82022-08-08 17:54:55 +02002432 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002433 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002434 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002435 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002436 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002437
2438 video_stream_encoder_->Stop();
2439}
2440
mflodmancc3d4422017-08-03 08:27:51 -07002441TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002442 EXPECT_TRUE(video_source_.has_sinks());
2443 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002444 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002445 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002446 EXPECT_FALSE(video_source_.has_sinks());
2447 EXPECT_TRUE(new_video_source.has_sinks());
2448
mflodmancc3d4422017-08-03 08:27:51 -07002449 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002450}
2451
mflodmancc3d4422017-08-03 08:27:51 -07002452TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002453 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002454 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002455 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002456 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002457}
2458
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002459class ResolutionAlignmentTest
2460 : public VideoStreamEncoderTest,
2461 public ::testing::WithParamInterface<
2462 ::testing::tuple<int, std::vector<double>>> {
2463 public:
2464 ResolutionAlignmentTest()
2465 : requested_alignment_(::testing::get<0>(GetParam())),
2466 scale_factors_(::testing::get<1>(GetParam())) {}
2467
2468 protected:
2469 const int requested_alignment_;
2470 const std::vector<double> scale_factors_;
2471};
2472
2473INSTANTIATE_TEST_SUITE_P(
2474 AlignmentAndScaleFactors,
2475 ResolutionAlignmentTest,
2476 ::testing::Combine(
2477 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2478 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2479 std::vector<double>{-1.0, -1.0},
2480 std::vector<double>{-1.0, -1.0, -1.0},
2481 std::vector<double>{4.0, 2.0, 1.0},
2482 std::vector<double>{9999.0, -1.0, 1.0},
2483 std::vector<double>{3.99, 2.01, 1.0},
2484 std::vector<double>{4.9, 1.7, 1.25},
2485 std::vector<double>{10.0, 4.0, 3.0},
2486 std::vector<double>{1.75, 3.5},
2487 std::vector<double>{1.5, 2.5},
2488 std::vector<double>{1.3, 1.0})));
2489
2490TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2491 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002492 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002493 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2494 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2495
2496 // Fill config with the scaling factor by which to reduce encoding size.
2497 const int num_streams = scale_factors_.size();
2498 VideoEncoderConfig config;
2499 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2500 for (int i = 0; i < num_streams; ++i) {
2501 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2502 }
2503 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002504 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002505 "VP8", /*max qp*/ 56, /*screencast*/ false,
2506 /*screenshare enabled*/ false);
2507 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2508
Henrik Boström381d1092020-05-12 18:49:07 +02002509 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002510 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2511 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002512 // Wait for all layers before triggering event.
2513 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002514
2515 // On the 1st frame, we should have initialized the encoder and
2516 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002517 int64_t timestamp_ms = kFrameIntervalMs;
2518 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2519 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002520 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002521
2522 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2523 // (It's up the to the encoder to potentially drop the previous frame,
2524 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002525 timestamp_ms += kFrameIntervalMs;
2526 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2527 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002528 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002529
Asa Persson606d3cb2021-10-04 10:07:11 +02002530 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002531 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2532 // Frame size should be a multiple of the requested alignment.
2533 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2534 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2535 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2536 // Aspect ratio should match.
2537 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2538 codec.height * codec.simulcastStream[i].width);
2539 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002540
2541 video_stream_encoder_->Stop();
2542}
2543
Jonathan Yubc771b72017-12-08 17:04:29 -08002544TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2545 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002546 const int kWidth = 1280;
2547 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002548
2549 // We rely on the automatic resolution adaptation, but we handle framerate
2550 // adaptation manually by mocking the stats proxy.
2551 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002552
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002553 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002554 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002555 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002556 video_stream_encoder_->SetSource(&video_source_,
2557 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002558 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002559 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002560 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002561 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2562
Jonathan Yubc771b72017-12-08 17:04:29 -08002563 // Adapt down as far as possible.
2564 rtc::VideoSinkWants last_wants;
2565 int64_t t = 1;
2566 int loop_count = 0;
2567 do {
2568 ++loop_count;
2569 last_wants = video_source_.sink_wants();
2570
2571 // Simulate the framerate we've been asked to adapt to.
2572 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2573 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2574 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2575 mock_stats.input_frame_rate = fps;
2576 stats_proxy_->SetMockStats(mock_stats);
2577
2578 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2579 sink_.WaitForEncodedFrame(t);
2580 t += frame_interval_ms;
2581
mflodmancc3d4422017-08-03 08:27:51 -07002582 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002583 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002584 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002585 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2586 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002587 } while (video_source_.sink_wants().max_pixel_count <
2588 last_wants.max_pixel_count ||
2589 video_source_.sink_wants().max_framerate_fps <
2590 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002591
Jonathan Yubc771b72017-12-08 17:04:29 -08002592 // Verify that we've adapted all the way down.
2593 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002594 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002595 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2596 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002597 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002598 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2599 *video_source_.last_sent_height());
2600 EXPECT_EQ(kMinBalancedFramerateFps,
2601 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002602
Jonathan Yubc771b72017-12-08 17:04:29 -08002603 // Adapt back up the same number of times we adapted down.
2604 for (int i = 0; i < loop_count - 1; ++i) {
2605 last_wants = video_source_.sink_wants();
2606
2607 // Simulate the framerate we've been asked to adapt to.
2608 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2609 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2610 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2611 mock_stats.input_frame_rate = fps;
2612 stats_proxy_->SetMockStats(mock_stats);
2613
2614 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2615 sink_.WaitForEncodedFrame(t);
2616 t += frame_interval_ms;
2617
Henrik Boström91aa7322020-04-28 12:24:33 +02002618 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002619 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002620 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002621 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2622 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002623 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2624 last_wants.max_pixel_count ||
2625 video_source_.sink_wants().max_framerate_fps >
2626 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002627 }
2628
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002629 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002630 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002631 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002632 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2633 EXPECT_EQ((loop_count - 1) * 2,
2634 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002635
mflodmancc3d4422017-08-03 08:27:51 -07002636 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002637}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002638
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002639TEST_F(VideoStreamEncoderTest,
2640 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02002641 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2642 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002643 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002644
2645 const int kFrameWidth = 1280;
2646 const int kFrameHeight = 720;
2647
2648 int64_t ntp_time = kFrameIntervalMs;
2649
2650 // Force an input frame rate to be available, or the adaptation call won't
2651 // know what framerate to adapt form.
2652 const int kInputFps = 30;
2653 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2654 stats.input_frame_rate = kInputFps;
2655 stats_proxy_->SetMockStats(stats);
2656
2657 video_source_.set_adaptation_enabled(true);
2658 video_stream_encoder_->SetSource(
2659 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002660 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002661 video_source_.IncomingCapturedFrame(
2662 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2663 sink_.WaitForEncodedFrame(ntp_time);
2664 ntp_time += kFrameIntervalMs;
2665
2666 // Trigger CPU overuse.
2667 video_stream_encoder_->TriggerCpuOveruse();
2668 video_source_.IncomingCapturedFrame(
2669 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2670 sink_.WaitForEncodedFrame(ntp_time);
2671 ntp_time += kFrameIntervalMs;
2672
2673 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2674 EXPECT_EQ(std::numeric_limits<int>::max(),
2675 video_source_.sink_wants().max_pixel_count);
2676 // Some framerate constraint should be set.
2677 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2678 EXPECT_LT(restricted_fps, kInputFps);
2679 video_source_.IncomingCapturedFrame(
2680 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2681 sink_.WaitForEncodedFrame(ntp_time);
2682 ntp_time += 100;
2683
Henrik Boström2671dac2020-05-19 16:29:09 +02002684 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002685 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2686 // Give the encoder queue time to process the change in degradation preference
2687 // by waiting for an encoded frame.
2688 video_source_.IncomingCapturedFrame(
2689 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2690 sink_.WaitForEncodedFrame(ntp_time);
2691 ntp_time += kFrameIntervalMs;
2692
2693 video_stream_encoder_->TriggerQualityLow();
2694 video_source_.IncomingCapturedFrame(
2695 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2696 sink_.WaitForEncodedFrame(ntp_time);
2697 ntp_time += kFrameIntervalMs;
2698
2699 // Some resolution constraint should be set.
2700 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2701 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2702 kFrameWidth * kFrameHeight);
2703 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2704
2705 int pixel_count = video_source_.sink_wants().max_pixel_count;
2706 // Triggering a CPU underuse should not change the sink wants since it has
2707 // not been overused for resolution since we changed degradation preference.
2708 video_stream_encoder_->TriggerCpuUnderuse();
2709 video_source_.IncomingCapturedFrame(
2710 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2711 sink_.WaitForEncodedFrame(ntp_time);
2712 ntp_time += kFrameIntervalMs;
2713 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2714 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2715
Evan Shrubsole64469032020-06-11 10:45:29 +02002716 // Change the degradation preference back. CPU underuse should not adapt since
2717 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002718 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002719 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2720 video_source_.IncomingCapturedFrame(
2721 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2722 sink_.WaitForEncodedFrame(ntp_time);
2723 ntp_time += 100;
2724 // Resolution adaptations is gone after changing degradation preference.
2725 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2726 EXPECT_EQ(std::numeric_limits<int>::max(),
2727 video_source_.sink_wants().max_pixel_count);
2728 // The fps adaptation from above is now back.
2729 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2730
2731 // Trigger CPU underuse.
2732 video_stream_encoder_->TriggerCpuUnderuse();
2733 video_source_.IncomingCapturedFrame(
2734 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2735 sink_.WaitForEncodedFrame(ntp_time);
2736 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002737 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2738
2739 // Trigger QP underuse, fps should return to normal.
2740 video_stream_encoder_->TriggerQualityHigh();
2741 video_source_.IncomingCapturedFrame(
2742 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2743 sink_.WaitForEncodedFrame(ntp_time);
2744 ntp_time += kFrameIntervalMs;
2745 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002746
2747 video_stream_encoder_->Stop();
2748}
2749
mflodmancc3d4422017-08-03 08:27:51 -07002750TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002751 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002752 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002753 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002754
sprangc5d62e22017-04-02 23:53:04 -07002755 const int kFrameWidth = 1280;
2756 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002757
Åsa Persson8c1bf952018-09-13 10:42:19 +02002758 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002759
kthelgason5e13d412016-12-01 03:59:51 -08002760 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002761 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002762 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002763 frame_timestamp += kFrameIntervalMs;
2764
perkj803d97f2016-11-01 11:45:46 -07002765 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002766 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002767 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002768 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002769 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002770 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002771
asapersson0944a802017-04-07 00:57:58 -07002772 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002773 // wanted resolution.
2774 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2775 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2776 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002777 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002778
2779 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002780 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002781 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002782 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002783 // Give the encoder queue time to process the change in degradation preference
2784 // by waiting for an encoded frame.
2785 new_video_source.IncomingCapturedFrame(
2786 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2787 sink_.WaitForEncodedFrame(frame_timestamp);
2788 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002789 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002790 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002791
sprangc5d62e22017-04-02 23:53:04 -07002792 // Force an input frame rate to be available, or the adaptation call won't
2793 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002794 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002795 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002796 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002797 stats_proxy_->SetMockStats(stats);
2798
mflodmancc3d4422017-08-03 08:27:51 -07002799 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002800 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002801 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002802 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002803 frame_timestamp += kFrameIntervalMs;
2804
2805 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002806 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002807 EXPECT_EQ(std::numeric_limits<int>::max(),
2808 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002809 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002810
asapersson02465b82017-04-10 01:12:52 -07002811 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002812 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2813 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002814 // Give the encoder queue time to process the change in degradation preference
2815 // by waiting for an encoded frame.
2816 new_video_source.IncomingCapturedFrame(
2817 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2818 sink_.WaitForEncodedFrame(frame_timestamp);
2819 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002820 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002821
mflodmancc3d4422017-08-03 08:27:51 -07002822 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002823 new_video_source.IncomingCapturedFrame(
2824 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002825 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002826 frame_timestamp += kFrameIntervalMs;
2827
2828 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002829 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002830
2831 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002832 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002833 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002834 // Give the encoder queue time to process the change in degradation preference
2835 // by waiting for an encoded frame.
2836 new_video_source.IncomingCapturedFrame(
2837 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2838 sink_.WaitForEncodedFrame(frame_timestamp);
2839 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002840 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2841 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002842 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002843 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002844
2845 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002846 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002847 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002848 // Give the encoder queue time to process the change in degradation preference
2849 // by waiting for an encoded frame.
2850 new_video_source.IncomingCapturedFrame(
2851 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2852 sink_.WaitForEncodedFrame(frame_timestamp);
2853 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002854 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2855 EXPECT_EQ(std::numeric_limits<int>::max(),
2856 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002857 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002858
mflodmancc3d4422017-08-03 08:27:51 -07002859 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002860}
2861
mflodmancc3d4422017-08-03 08:27:51 -07002862TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002863 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002864 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002865
asaperssonfab67072017-04-04 05:51:49 -07002866 const int kWidth = 1280;
2867 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002868 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002869 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002870 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2871 EXPECT_FALSE(stats.bw_limited_resolution);
2872 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2873
2874 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002875 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002876 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002877 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002878
2879 stats = stats_proxy_->GetStats();
2880 EXPECT_TRUE(stats.bw_limited_resolution);
2881 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2882
2883 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002884 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002885 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002886 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002887
2888 stats = stats_proxy_->GetStats();
2889 EXPECT_FALSE(stats.bw_limited_resolution);
2890 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2891 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2892
mflodmancc3d4422017-08-03 08:27:51 -07002893 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002894}
2895
mflodmancc3d4422017-08-03 08:27:51 -07002896TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002897 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002898 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002899
2900 const int kWidth = 1280;
2901 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002902 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002903 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002904 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2905 EXPECT_FALSE(stats.cpu_limited_resolution);
2906 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2907
2908 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002909 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002910 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002911 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002912
2913 stats = stats_proxy_->GetStats();
2914 EXPECT_TRUE(stats.cpu_limited_resolution);
2915 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2916
2917 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002918 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002919 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002920 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002921
2922 stats = stats_proxy_->GetStats();
2923 EXPECT_FALSE(stats.cpu_limited_resolution);
2924 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002925 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002926
mflodmancc3d4422017-08-03 08:27:51 -07002927 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002928}
2929
mflodmancc3d4422017-08-03 08:27:51 -07002930TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002931 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002932 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002933
asaperssonfab67072017-04-04 05:51:49 -07002934 const int kWidth = 1280;
2935 const int kHeight = 720;
2936 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002937 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002938 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002939 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002940 EXPECT_FALSE(stats.cpu_limited_resolution);
2941 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2942
asaperssonfab67072017-04-04 05:51:49 -07002943 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002944 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002945 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002946 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002947 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002948 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002949 EXPECT_TRUE(stats.cpu_limited_resolution);
2950 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2951
2952 // Set new source with adaptation still enabled.
2953 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002954 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002955 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002956
asaperssonfab67072017-04-04 05:51:49 -07002957 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002958 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002959 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002960 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002961 EXPECT_TRUE(stats.cpu_limited_resolution);
2962 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2963
2964 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002965 video_stream_encoder_->SetSource(&new_video_source,
2966 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002967
asaperssonfab67072017-04-04 05:51:49 -07002968 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002969 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002970 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002971 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002972 EXPECT_FALSE(stats.cpu_limited_resolution);
2973 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2974
2975 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002976 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002977 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002978
asaperssonfab67072017-04-04 05:51:49 -07002979 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002980 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002981 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002982 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002983 EXPECT_TRUE(stats.cpu_limited_resolution);
2984 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2985
asaperssonfab67072017-04-04 05:51:49 -07002986 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002987 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002988 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002989 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002990 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002991 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002992 EXPECT_FALSE(stats.cpu_limited_resolution);
2993 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002994 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002995
mflodmancc3d4422017-08-03 08:27:51 -07002996 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002997}
2998
mflodmancc3d4422017-08-03 08:27:51 -07002999TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02003000 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003001 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003002
asaperssonfab67072017-04-04 05:51:49 -07003003 const int kWidth = 1280;
3004 const int kHeight = 720;
3005 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003006 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003007 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003008 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003009 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003010 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003011
3012 // Set new source with adaptation still enabled.
3013 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003014 video_stream_encoder_->SetSource(&new_video_source,
3015 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08003016
asaperssonfab67072017-04-04 05:51:49 -07003017 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003018 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08003019 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003020 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003021 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003022 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003023
asaperssonfab67072017-04-04 05:51:49 -07003024 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003025 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003026 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003027 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003028 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003029 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003030 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003031 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003032
asaperssonfab67072017-04-04 05:51:49 -07003033 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003034 video_stream_encoder_->SetSource(&new_video_source,
3035 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08003036
asaperssonfab67072017-04-04 05:51:49 -07003037 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003038 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003039 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003040 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003041 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003042 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003043
asapersson02465b82017-04-10 01:12:52 -07003044 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07003045 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003046 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003047
asaperssonfab67072017-04-04 05:51:49 -07003048 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003049 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08003050 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003051 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003052 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003053 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
3054 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003055
mflodmancc3d4422017-08-03 08:27:51 -07003056 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003057}
3058
mflodmancc3d4422017-08-03 08:27:51 -07003059TEST_F(VideoStreamEncoderTest,
3060 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02003061 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003062 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07003063
3064 const int kWidth = 1280;
3065 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02003066 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07003067 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003068 video_source_.IncomingCapturedFrame(
3069 CreateFrame(timestamp_ms, kWidth, kHeight));
3070 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003071 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3072 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3073 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3074
3075 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003076 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003077 timestamp_ms += kFrameIntervalMs;
3078 video_source_.IncomingCapturedFrame(
3079 CreateFrame(timestamp_ms, kWidth, kHeight));
3080 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003081 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3082 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3083 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3084
3085 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07003086 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003087 timestamp_ms += kFrameIntervalMs;
3088 video_source_.IncomingCapturedFrame(
3089 CreateFrame(timestamp_ms, kWidth, kHeight));
3090 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003091 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3092 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3093 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3094
Niels Möller4db138e2018-04-19 09:04:13 +02003095 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07003096 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003097
3098 VideoEncoderConfig video_encoder_config;
3099 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3100 // Make format different, to force recreation of encoder.
3101 video_encoder_config.video_format.parameters["foo"] = "foo";
3102 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003103 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003104 timestamp_ms += kFrameIntervalMs;
3105 video_source_.IncomingCapturedFrame(
3106 CreateFrame(timestamp_ms, kWidth, kHeight));
3107 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003108 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3109 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3110 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3111
mflodmancc3d4422017-08-03 08:27:51 -07003112 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003113}
3114
mflodmancc3d4422017-08-03 08:27:51 -07003115TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003116 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003117 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003118 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003119
3120 const int kWidth = 1280;
3121 const int kHeight = 720;
3122 int sequence = 1;
3123
3124 // Enable BALANCED preference, no initial limitation.
3125 test::FrameForwarder source;
3126 video_stream_encoder_->SetSource(&source,
3127 webrtc::DegradationPreference::BALANCED);
3128 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3129 WaitForEncodedFrame(sequence++);
3130 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3131 EXPECT_FALSE(stats.cpu_limited_resolution);
3132 EXPECT_FALSE(stats.cpu_limited_framerate);
3133 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3134
3135 // Trigger CPU overuse, should now adapt down.
3136 video_stream_encoder_->TriggerCpuOveruse();
3137 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3138 WaitForEncodedFrame(sequence++);
3139 stats = stats_proxy_->GetStats();
3140 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3141
3142 // Set new degradation preference should clear restrictions since we changed
3143 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003144 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003145 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3146 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3147 WaitForEncodedFrame(sequence++);
3148 stats = stats_proxy_->GetStats();
3149 EXPECT_FALSE(stats.cpu_limited_resolution);
3150 EXPECT_FALSE(stats.cpu_limited_framerate);
3151 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3152
3153 // Force an input frame rate to be available, or the adaptation call won't
3154 // know what framerate to adapt from.
3155 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3156 mock_stats.input_frame_rate = 30;
3157 stats_proxy_->SetMockStats(mock_stats);
3158 video_stream_encoder_->TriggerCpuOveruse();
3159 stats_proxy_->ResetMockStats();
3160 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3161 WaitForEncodedFrame(sequence++);
3162
3163 // We have now adapted once.
3164 stats = stats_proxy_->GetStats();
3165 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3166
3167 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003168 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3169 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003170 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3171 WaitForEncodedFrame(sequence++);
3172 stats = stats_proxy_->GetStats();
3173 EXPECT_FALSE(stats.cpu_limited_resolution);
3174 EXPECT_FALSE(stats.cpu_limited_framerate);
3175 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3176
3177 video_stream_encoder_->Stop();
3178}
3179
3180TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003181 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003182 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003183 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003184
asapersson0944a802017-04-07 00:57:58 -07003185 const int kWidth = 1280;
3186 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003187 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003188
asaperssonfab67072017-04-04 05:51:49 -07003189 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003190 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003191 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003192 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003193 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003194 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3195
asapersson02465b82017-04-10 01:12:52 -07003196 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003197 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003198 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003199 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003200 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003201 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003202 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003203 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3204
3205 // Set new source with adaptation still enabled.
3206 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003207 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003208 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003209
3210 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003211 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003212 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003213 stats = stats_proxy_->GetStats();
3214 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003215 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003216 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3217
sprangc5d62e22017-04-02 23:53:04 -07003218 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003219 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003220 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003221 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003222 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003223 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003224 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003225 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003226 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003227 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003228 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3229
sprangc5d62e22017-04-02 23:53:04 -07003230 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003231 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003232 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3233 mock_stats.input_frame_rate = 30;
3234 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003235 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003236 stats_proxy_->ResetMockStats();
3237
3238 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003239 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003240 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003241
3242 // Framerate now adapted.
3243 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003244 EXPECT_FALSE(stats.cpu_limited_resolution);
3245 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003246 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3247
3248 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003249 video_stream_encoder_->SetSource(&new_video_source,
3250 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003251 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003252 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003253 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003254
3255 stats = stats_proxy_->GetStats();
3256 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003257 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003258 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3259
3260 // Try to trigger overuse. Should not succeed.
3261 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003262 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003263 stats_proxy_->ResetMockStats();
3264
3265 stats = stats_proxy_->GetStats();
3266 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003267 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003268 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3269
3270 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003271 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003272 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003273 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003274 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003275 stats = stats_proxy_->GetStats();
3276 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003277 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003278 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003279
3280 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003281 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003282 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003283 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003284 stats = stats_proxy_->GetStats();
3285 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003286 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003287 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3288
3289 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003290 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003291 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003292 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003293 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003294 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003295 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003296 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003297 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003298 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003299 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3300
3301 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003302 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003303 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003304 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003305 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003306 stats = stats_proxy_->GetStats();
3307 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003308 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003309 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003310 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003311
mflodmancc3d4422017-08-03 08:27:51 -07003312 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003313}
3314
mflodmancc3d4422017-08-03 08:27:51 -07003315TEST_F(VideoStreamEncoderTest,
3316 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003317 const int kWidth = 1280;
3318 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003319 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003320 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003321
asaperssonfab67072017-04-04 05:51:49 -07003322 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003323 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003324
asaperssonfab67072017-04-04 05:51:49 -07003325 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003326 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003327
asaperssonfab67072017-04-04 05:51:49 -07003328 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003329 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003330
asaperssonfab67072017-04-04 05:51:49 -07003331 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003332 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003333
kthelgason876222f2016-11-29 01:44:11 -08003334 // Expect a scale down.
3335 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003336 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003337
asapersson02465b82017-04-10 01:12:52 -07003338 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003339 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003340 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003341 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003342
asaperssonfab67072017-04-04 05:51:49 -07003343 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003344 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003345 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003346 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003347
asaperssonfab67072017-04-04 05:51:49 -07003348 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003349 EXPECT_EQ(std::numeric_limits<int>::max(),
3350 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003351
asaperssonfab67072017-04-04 05:51:49 -07003352 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003353 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003354 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003355 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003356
asapersson02465b82017-04-10 01:12:52 -07003357 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003358 EXPECT_EQ(std::numeric_limits<int>::max(),
3359 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003360
mflodmancc3d4422017-08-03 08:27:51 -07003361 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003362}
3363
mflodmancc3d4422017-08-03 08:27:51 -07003364TEST_F(VideoStreamEncoderTest,
3365 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003366 const int kWidth = 1280;
3367 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003368 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003369 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003370
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003371 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003372 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003373 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003374 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003375
3376 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003377 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003378 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003379 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3380 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3381
3382 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003383 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003384 EXPECT_THAT(source.sink_wants(),
3385 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003386 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3387 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3388 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3389
3390 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003391 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003392 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3393 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3394 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3395
mflodmancc3d4422017-08-03 08:27:51 -07003396 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003397}
3398
mflodmancc3d4422017-08-03 08:27:51 -07003399TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003400 const int kWidth = 1280;
3401 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003402 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003403 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003404
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003405 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003406 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003407 video_stream_encoder_->SetSource(&source,
3408 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003409 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3410 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003411 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003412
3413 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003414 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003415 EXPECT_THAT(source.sink_wants(),
3416 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003417 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3418 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3419 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3420
3421 // Trigger adapt down for same input resolution, expect no change.
3422 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3423 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003424 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003425 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3426 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3427 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3428
3429 // Trigger adapt down for larger input resolution, expect no change.
3430 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3431 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003432 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003433 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3434 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3435 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3436
mflodmancc3d4422017-08-03 08:27:51 -07003437 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003438}
3439
mflodmancc3d4422017-08-03 08:27:51 -07003440TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003441 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3442 const int kWidth = 640;
3443 const int kHeight = 360;
3444 const int64_t kFrameIntervalMs = 150;
3445 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003446 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003447
3448 // Enable BALANCED preference, no initial limitation.
3449 AdaptingFrameForwarder source(&time_controller_);
3450 source.set_adaptation_enabled(true);
3451 video_stream_encoder_->SetSource(&source,
3452 webrtc::DegradationPreference::BALANCED);
3453
3454 int64_t timestamp_ms = kFrameIntervalMs;
3455 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3456 sink_.WaitForEncodedFrame(kWidth, kHeight);
3457 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3458 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3459 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3460 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3461
3462 // Trigger adapt down, expect reduced fps (640x360@15fps).
3463 video_stream_encoder_->TriggerQualityLow();
3464 timestamp_ms += kFrameIntervalMs;
3465 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3466 sink_.WaitForEncodedFrame(timestamp_ms);
3467 EXPECT_THAT(source.sink_wants(),
3468 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3469 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3470 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3471 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3472
3473 // Source requests 270p, expect reduced resolution (480x270@15fps).
3474 source.OnOutputFormatRequest(480, 270);
3475 timestamp_ms += kFrameIntervalMs;
3476 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3477 WaitForEncodedFrame(480, 270);
3478 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3479
3480 // Trigger adapt down, expect reduced fps (480x270@10fps).
3481 video_stream_encoder_->TriggerQualityLow();
3482 timestamp_ms += kFrameIntervalMs;
3483 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3484 sink_.WaitForEncodedFrame(timestamp_ms);
3485 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3486 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3487 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3488 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3489
3490 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3491 source.OnOutputFormatRequest(320, 180);
3492 timestamp_ms += kFrameIntervalMs;
3493 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3494 WaitForEncodedFrame(320, 180);
3495 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3496
3497 // Trigger adapt down, expect reduced fps (320x180@7fps).
3498 video_stream_encoder_->TriggerQualityLow();
3499 timestamp_ms += kFrameIntervalMs;
3500 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3501 sink_.WaitForEncodedFrame(timestamp_ms);
3502 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3503 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3504 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3505 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3506
3507 // Source requests VGA, expect increased resolution (640x360@7fps).
3508 source.OnOutputFormatRequest(640, 360);
3509 timestamp_ms += kFrameIntervalMs;
3510 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3511 WaitForEncodedFrame(timestamp_ms);
3512 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3513
3514 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3515 video_stream_encoder_->TriggerQualityHigh();
3516 timestamp_ms += kFrameIntervalMs;
3517 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3518 WaitForEncodedFrame(timestamp_ms);
3519 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3520 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3521 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3522 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3523
3524 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3525 video_stream_encoder_->TriggerQualityHigh();
3526 timestamp_ms += kFrameIntervalMs;
3527 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3528 WaitForEncodedFrame(timestamp_ms);
3529 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3530 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3531 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3532 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3533
3534 // Trigger adapt up, expect increased fps (640x360@maxfps).
3535 video_stream_encoder_->TriggerQualityHigh();
3536 timestamp_ms += kFrameIntervalMs;
3537 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3538 WaitForEncodedFrame(timestamp_ms);
3539 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3540 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3541 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3542 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3543
3544 video_stream_encoder_->Stop();
3545}
3546
3547TEST_F(VideoStreamEncoderTest,
3548 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3549 const int kWidth = 1280;
3550 const int kHeight = 720;
3551 const int64_t kFrameIntervalMs = 150;
3552 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003553 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003554
3555 // Enable BALANCED preference, no initial limitation.
3556 AdaptingFrameForwarder source(&time_controller_);
3557 source.set_adaptation_enabled(true);
3558 video_stream_encoder_->SetSource(&source,
3559 webrtc::DegradationPreference::BALANCED);
3560
3561 int64_t timestamp_ms = kFrameIntervalMs;
3562 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3563 sink_.WaitForEncodedFrame(kWidth, kHeight);
3564 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3565 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3566 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3567 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3568
3569 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3570 video_stream_encoder_->TriggerQualityLow();
3571 timestamp_ms += kFrameIntervalMs;
3572 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3573 sink_.WaitForEncodedFrame(timestamp_ms);
3574 EXPECT_THAT(source.sink_wants(),
3575 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3576 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3577 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3578 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3579
3580 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3581 video_stream_encoder_->TriggerQualityLow();
3582 timestamp_ms += kFrameIntervalMs;
3583 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3584 sink_.WaitForEncodedFrame(timestamp_ms);
3585 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3586 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3587 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3588 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3589
3590 // Trigger adapt down, expect reduced fps (640x360@15fps).
3591 video_stream_encoder_->TriggerQualityLow();
3592 timestamp_ms += kFrameIntervalMs;
3593 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3594 WaitForEncodedFrame(timestamp_ms);
3595 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3596 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3597 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3598 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3599
3600 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3601 source.OnOutputFormatRequest(320, 180);
3602 timestamp_ms += kFrameIntervalMs;
3603 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3604 WaitForEncodedFrame(320, 180);
3605 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3606 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3607
3608 // Trigger adapt down, expect reduced fps (320x180@7fps).
3609 video_stream_encoder_->TriggerCpuOveruse();
3610 timestamp_ms += kFrameIntervalMs;
3611 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3612 WaitForEncodedFrame(timestamp_ms);
3613 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3614 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3615 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3616 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3617 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3618 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3619 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3620
3621 // Source requests HD, expect increased resolution (640x360@7fps).
3622 source.OnOutputFormatRequest(1280, 720);
3623 timestamp_ms += kFrameIntervalMs;
3624 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3625 WaitForEncodedFrame(timestamp_ms);
3626 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3627 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3628
3629 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3630 video_stream_encoder_->TriggerCpuUnderuse();
3631 timestamp_ms += kFrameIntervalMs;
3632 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3633 WaitForEncodedFrame(timestamp_ms);
3634 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3635 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3636 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3637 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3638 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3639 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3640 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3641
3642 // Trigger adapt up, expect increased fps (640x360@maxfps).
3643 video_stream_encoder_->TriggerQualityHigh();
3644 video_stream_encoder_->TriggerCpuUnderuse();
3645 timestamp_ms += kFrameIntervalMs;
3646 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3647 WaitForEncodedFrame(timestamp_ms);
3648 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3649 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3650 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3651 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3652 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3653 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3654 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3655
3656 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3657 video_stream_encoder_->TriggerQualityHigh();
3658 video_stream_encoder_->TriggerCpuUnderuse();
3659 timestamp_ms += kFrameIntervalMs;
3660 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3661 WaitForEncodedFrame(timestamp_ms);
3662 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3663 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3664 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3665 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3666 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3667 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3668 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3669
3670 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3671 video_stream_encoder_->TriggerQualityHigh();
3672 video_stream_encoder_->TriggerCpuUnderuse();
3673 timestamp_ms += kFrameIntervalMs;
3674 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3675 WaitForEncodedFrame(timestamp_ms);
3676 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3677 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3678 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3679 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3680 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3681 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3682 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3683
3684 video_stream_encoder_->Stop();
3685}
3686
3687TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003688 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003689 const int kWidth = 1280;
3690 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003691 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003692 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003693
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003694 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003695 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003696 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003697 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003698
3699 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003700 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003701 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003702 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3703 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3704
3705 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003706 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003707 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003708 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3709 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3710
mflodmancc3d4422017-08-03 08:27:51 -07003711 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003712}
3713
mflodmancc3d4422017-08-03 08:27:51 -07003714TEST_F(VideoStreamEncoderTest,
3715 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003716 const int kWidth = 1280;
3717 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003718 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003719 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003720
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003721 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003722 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003723 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003724 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003725
3726 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003727 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003728 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003729 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003730 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3731
3732 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003733 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003734 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003735 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003736 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3737
mflodmancc3d4422017-08-03 08:27:51 -07003738 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003739}
3740
mflodmancc3d4422017-08-03 08:27:51 -07003741TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003742 const int kWidth = 1280;
3743 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003744 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003745 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003746
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003747 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003748 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003749 video_stream_encoder_->SetSource(&source,
3750 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003751
3752 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3753 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003754 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003755 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3756 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3757 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3758
3759 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003760 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003761 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003762 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3763 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3764 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3765
mflodmancc3d4422017-08-03 08:27:51 -07003766 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003767}
3768
mflodmancc3d4422017-08-03 08:27:51 -07003769TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003770 const int kWidth = 1280;
3771 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003772 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003773 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003774
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003775 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003776 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003777 video_stream_encoder_->SetSource(&source,
3778 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003779
3780 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3781 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003782 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003783 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3784 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3785 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3786
3787 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003788 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003789 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003790 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3791 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3792 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3793
mflodmancc3d4422017-08-03 08:27:51 -07003794 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003795}
3796
mflodmancc3d4422017-08-03 08:27:51 -07003797TEST_F(VideoStreamEncoderTest,
3798 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003799 const int kWidth = 1280;
3800 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003801 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003802 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003803
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003804 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003805 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003806 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003807 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003808 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003809
3810 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003811 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003812 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003813 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3814 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3815
3816 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003817 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003818 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003819 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003820 EXPECT_THAT(source.sink_wants(),
3821 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003822 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3823 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3824
3825 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003826 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003827 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003828 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3829 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3830 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3831
mflodmancc3d4422017-08-03 08:27:51 -07003832 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003833}
3834
mflodmancc3d4422017-08-03 08:27:51 -07003835TEST_F(VideoStreamEncoderTest,
3836 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003837 const int kWidth = 1280;
3838 const int kHeight = 720;
3839 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003840 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003841 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003842
3843 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3844 stats.input_frame_rate = kInputFps;
3845 stats_proxy_->SetMockStats(stats);
3846
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003847 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003848 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3849 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003850 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003851
3852 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003853 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003854 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3855 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003856 EXPECT_THAT(video_source_.sink_wants(),
3857 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003858
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003859 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003860 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003861 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003862 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003863 // Give the encoder queue time to process the change in degradation preference
3864 // by waiting for an encoded frame.
3865 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3866 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003867 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003868
3869 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003870 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003871 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3872 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003873 EXPECT_THAT(new_video_source.sink_wants(),
3874 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003875
3876 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003877 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003878 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003879
mflodmancc3d4422017-08-03 08:27:51 -07003880 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003881}
3882
mflodmancc3d4422017-08-03 08:27:51 -07003883TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003884 const int kWidth = 1280;
3885 const int kHeight = 720;
3886 const size_t kNumFrames = 10;
3887
Henrik Boström381d1092020-05-12 18:49:07 +02003888 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003889 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003890
asaperssond0de2952017-04-21 01:47:31 -07003891 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003892 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003893 video_source_.set_adaptation_enabled(true);
3894
3895 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3896 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3897
3898 int downscales = 0;
3899 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003900 video_source_.IncomingCapturedFrame(
3901 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3902 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003903
asaperssonfab67072017-04-04 05:51:49 -07003904 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003905 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003906 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003907 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003908
3909 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3910 ++downscales;
3911
3912 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3913 EXPECT_EQ(downscales,
3914 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3915 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003916 }
mflodmancc3d4422017-08-03 08:27:51 -07003917 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003918}
3919
mflodmancc3d4422017-08-03 08:27:51 -07003920TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003921 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3922 const int kWidth = 1280;
3923 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003924 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003925 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003926
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003927 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003928 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003929 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003930 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003931 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003932
Åsa Persson8c1bf952018-09-13 10:42:19 +02003933 int64_t timestamp_ms = kFrameIntervalMs;
3934 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003935 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003936 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003937 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3938 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3939
3940 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003941 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003942 timestamp_ms += kFrameIntervalMs;
3943 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3944 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003945 EXPECT_THAT(source.sink_wants(),
3946 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003947 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3948 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3949
3950 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003951 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003952 timestamp_ms += kFrameIntervalMs;
3953 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003954 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003955 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003956 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3957 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3958
3959 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003960 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003961 timestamp_ms += kFrameIntervalMs;
3962 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3963 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003964 EXPECT_THAT(source.sink_wants(),
3965 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003966 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3967 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3968
3969 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003970 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003971 timestamp_ms += kFrameIntervalMs;
3972 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003973 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003974 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003975 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3976 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3977
mflodmancc3d4422017-08-03 08:27:51 -07003978 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003979}
3980
mflodmancc3d4422017-08-03 08:27:51 -07003981TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003982 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3983 const int kWidth = 1280;
3984 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003985 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003986 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003987
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003988 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003989 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003990 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003991 video_stream_encoder_->SetSource(&source,
3992 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003993
Åsa Persson8c1bf952018-09-13 10:42:19 +02003994 int64_t timestamp_ms = kFrameIntervalMs;
3995 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003996 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003997 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003998 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3999 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4000
4001 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07004002 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004003 timestamp_ms += kFrameIntervalMs;
4004 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4005 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004006 EXPECT_THAT(source.sink_wants(),
4007 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004008 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4009 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4010
4011 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07004012 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004013 timestamp_ms += kFrameIntervalMs;
4014 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07004015 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004016 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004017 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4018 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4019
4020 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07004021 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004022 timestamp_ms += kFrameIntervalMs;
4023 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4024 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004025 EXPECT_THAT(source.sink_wants(),
4026 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004027 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4028 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4029
4030 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07004031 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004032 timestamp_ms += kFrameIntervalMs;
4033 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07004034 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004035 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004036 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4037 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4038
mflodmancc3d4422017-08-03 08:27:51 -07004039 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004040}
4041
Sergey Silkin41c650b2019-10-14 13:12:19 +02004042TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
4043 fake_encoder_.SetResolutionBitrateLimits(
4044 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4045
Henrik Boström381d1092020-05-12 18:49:07 +02004046 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004047 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4048 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4049 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4050 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004051
4052 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004053 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004054 source.set_adaptation_enabled(true);
4055 video_stream_encoder_->SetSource(
4056 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4057
4058 // Insert 720p frame.
4059 int64_t timestamp_ms = kFrameIntervalMs;
4060 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4061 WaitForEncodedFrame(1280, 720);
4062
4063 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02004064 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004065 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4066 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4067 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4068 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004069 video_stream_encoder_->TriggerQualityLow();
4070
4071 // Insert 720p frame. It should be downscaled and encoded.
4072 timestamp_ms += kFrameIntervalMs;
4073 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4074 WaitForEncodedFrame(960, 540);
4075
4076 // Trigger adapt up. Higher resolution should not be requested duo to lack
4077 // of bitrate.
4078 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004079 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02004080
4081 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02004082 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004083 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4084 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4085 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4086 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004087
4088 // Trigger adapt up. Higher resolution should be requested.
4089 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004090 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02004091
4092 video_stream_encoder_->Stop();
4093}
4094
4095TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
4096 fake_encoder_.SetResolutionBitrateLimits(
4097 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4098
4099 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02004100 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004101 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4102 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4103 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4104 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004105
4106 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004107 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004108 source.set_adaptation_enabled(true);
4109 video_stream_encoder_->SetSource(
4110 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4111
4112 // Insert 720p frame. It should be dropped and lower resolution should be
4113 // requested.
4114 int64_t timestamp_ms = kFrameIntervalMs;
4115 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4116 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004117 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004118
4119 // Insert 720p frame. It should be downscaled and encoded.
4120 timestamp_ms += kFrameIntervalMs;
4121 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4122 WaitForEncodedFrame(960, 540);
4123
4124 video_stream_encoder_->Stop();
4125}
4126
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004127class BalancedDegradationTest : public VideoStreamEncoderTest {
4128 protected:
4129 void SetupTest() {
4130 // Reset encoder for field trials to take effect.
4131 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02004132 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004133
4134 // Enable BALANCED preference.
4135 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02004136 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4137 }
4138
Asa Persson606d3cb2021-10-04 10:07:11 +02004139 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02004140 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004141 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004142 }
4143
Åsa Persson45b176f2019-09-30 11:19:05 +02004144 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004145 timestamp_ms_ += kFrameIntervalMs;
4146 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02004147 }
4148
4149 void InsertFrameAndWaitForEncoded() {
4150 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004151 sink_.WaitForEncodedFrame(timestamp_ms_);
4152 }
4153
4154 const int kWidth = 640; // pixels:640x360=230400
4155 const int kHeight = 360;
4156 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4157 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004158 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004159};
4160
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004161TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004162 test::ScopedKeyValueConfig field_trials(
4163 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004164 "WebRTC-Video-BalancedDegradationSettings/"
4165 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4166 SetupTest();
4167
4168 // Force input frame rate.
4169 const int kInputFps = 24;
4170 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4171 stats.input_frame_rate = kInputFps;
4172 stats_proxy_->SetMockStats(stats);
4173
Åsa Persson45b176f2019-09-30 11:19:05 +02004174 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004175 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004176
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004177 // Trigger adapt down, expect scaled down framerate and resolution,
4178 // since Fps diff (input-requested:0) < threshold.
4179 video_stream_encoder_->TriggerQualityLow();
4180 EXPECT_THAT(source_.sink_wants(),
4181 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004182
4183 video_stream_encoder_->Stop();
4184}
4185
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004186TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004187 test::ScopedKeyValueConfig field_trials(
4188 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004189 "WebRTC-Video-BalancedDegradationSettings/"
4190 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4191 SetupTest();
4192
4193 // Force input frame rate.
4194 const int kInputFps = 25;
4195 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4196 stats.input_frame_rate = kInputFps;
4197 stats_proxy_->SetMockStats(stats);
4198
Åsa Persson45b176f2019-09-30 11:19:05 +02004199 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004200 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004201
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004202 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4203 // Fps diff (input-requested:1) == threshold.
4204 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004205 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004206
4207 video_stream_encoder_->Stop();
4208}
4209
4210TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004211 test::ScopedKeyValueConfig field_trials(
4212 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004213 "WebRTC-Video-BalancedDegradationSettings/"
4214 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4215 SetupTest();
4216
4217 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4218
Åsa Persson45b176f2019-09-30 11:19:05 +02004219 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004220 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004221
4222 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4223 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004224 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004225
4226 video_stream_encoder_->Stop();
4227}
4228
Åsa Perssonccfb3402019-09-25 15:13:04 +02004229TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004230 test::ScopedKeyValueConfig field_trials(
4231 field_trials_,
Åsa Persson1b247f12019-08-14 17:26:39 +02004232 "WebRTC-Video-BalancedDegradationSettings/"
4233 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004234 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004235
Asa Persson606d3cb2021-10-04 10:07:11 +02004236 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4237 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4238 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004239
Åsa Persson45b176f2019-09-30 11:19:05 +02004240 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004241 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004242 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4243
4244 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4245 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004246 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004247 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004248 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4249
4250 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4251 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004252 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004253 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004254 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4255
Åsa Persson30ab0152019-08-27 12:22:33 +02004256 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4257 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004258 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004259 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004260 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004261 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4262
4263 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004264 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004265 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004266 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004267
Åsa Persson30ab0152019-08-27 12:22:33 +02004268 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004269 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004270 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004271 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004272 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004273 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4274
4275 video_stream_encoder_->Stop();
4276}
4277
Åsa Perssonccfb3402019-09-25 15:13:04 +02004278TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004279 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004280 test::ScopedKeyValueConfig field_trials(
4281 field_trials_,
Åsa Persson45b176f2019-09-30 11:19:05 +02004282 "WebRTC-Video-BalancedDegradationSettings/"
4283 "pixels:57600|129600|230400,fps:7|24|24/");
4284 SetupTest();
Asa Persson606d3cb2021-10-04 10:07:11 +02004285 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 11:19:05 +02004286
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004287 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004288
4289 // Insert frame, expect scaled down:
4290 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4291 InsertFrame();
Markus Handell2cfc1af2022-08-19 08:16:48 +00004292 EXPECT_FALSE(WaitForFrame(TimeDelta::Seconds(1)));
Åsa Persson45b176f2019-09-30 11:19:05 +02004293 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4294 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4295
4296 // Insert frame, expect scaled down:
4297 // resolution (320x180@24fps).
4298 InsertFrame();
Markus Handell2cfc1af2022-08-19 08:16:48 +00004299 EXPECT_FALSE(WaitForFrame(TimeDelta::Seconds(1)));
Åsa Persson45b176f2019-09-30 11:19:05 +02004300 EXPECT_LT(source_.sink_wants().max_pixel_count,
4301 source_.last_wants().max_pixel_count);
4302 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4303
4304 // Frame should not be dropped (min pixels per frame reached).
4305 InsertFrameAndWaitForEncoded();
4306
4307 video_stream_encoder_->Stop();
4308}
4309
4310TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004311 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004312 test::ScopedKeyValueConfig field_trials(
4313 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004314 "WebRTC-Video-BalancedDegradationSettings/"
4315 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004316 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004317
Asa Persson606d3cb2021-10-04 10:07:11 +02004318 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4319 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4320 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004321
Åsa Persson45b176f2019-09-30 11:19:05 +02004322 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004323 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004324 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4325
4326 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4327 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004328 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004329 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004330 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4331
4332 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4333 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004334 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004335 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004336 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4337
4338 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4339 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004340 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004341 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004342 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4343
Åsa Persson30ab0152019-08-27 12:22:33 +02004344 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4345 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004346 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004347 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004348 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4349
4350 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4351 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004352 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004353 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4354
4355 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004356 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004357 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004358 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004359 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004360 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4361
4362 video_stream_encoder_->Stop();
4363}
4364
Åsa Perssonccfb3402019-09-25 15:13:04 +02004365TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004366 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004367 test::ScopedKeyValueConfig field_trials(
4368 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004369 "WebRTC-Video-BalancedDegradationSettings/"
4370 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004371 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004372
Asa Persson606d3cb2021-10-04 10:07:11 +02004373 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4374 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4375 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4376 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4377 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004378
Åsa Persson45b176f2019-09-30 11:19:05 +02004379 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004380 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004381 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4382
4383 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4384 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004385 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004386 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004387 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4388
4389 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4390 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004391 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004392 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004393 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4394
4395 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4396 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004397 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004398 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004399 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4400
4401 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4402 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004403 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004404 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4405
4406 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004407 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004408 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004409 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004410 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004411 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4412
4413 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004414 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004415 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004416 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004417 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4418
4419 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004420 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004421 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004422 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004423 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004424 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4425
Åsa Persson1b247f12019-08-14 17:26:39 +02004426 video_stream_encoder_->Stop();
4427}
4428
mflodmancc3d4422017-08-03 08:27:51 -07004429TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004430 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4431 const int kWidth = 1280;
4432 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004433 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004434 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004435
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004436 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004437 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004438 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004439 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004440 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004441
Åsa Persson8c1bf952018-09-13 10:42:19 +02004442 int64_t timestamp_ms = kFrameIntervalMs;
4443 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004444 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004445 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004446 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4447 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4448 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4449 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4450
4451 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004452 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004453 timestamp_ms += kFrameIntervalMs;
4454 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4455 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004456 EXPECT_THAT(source.sink_wants(),
4457 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004458 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4459 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4460 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4461 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4462
4463 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004464 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004465 timestamp_ms += kFrameIntervalMs;
4466 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4467 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004468 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004469 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4470 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4471 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4472 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4473
Jonathan Yubc771b72017-12-08 17:04:29 -08004474 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004475 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004476 timestamp_ms += kFrameIntervalMs;
4477 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4478 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004479 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004480 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4481 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004482 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004483 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4484
Jonathan Yubc771b72017-12-08 17:04:29 -08004485 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004486 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004487 timestamp_ms += kFrameIntervalMs;
4488 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4489 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004490 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004491 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004492 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4493 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4494 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4495 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4496
Jonathan Yubc771b72017-12-08 17:04:29 -08004497 // Trigger quality adapt down, expect no change (min resolution reached).
4498 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004499 timestamp_ms += kFrameIntervalMs;
4500 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4501 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004502 EXPECT_THAT(source.sink_wants(), FpsMax());
4503 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004504 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4505 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4506 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4507 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4508
Evan Shrubsole64469032020-06-11 10:45:29 +02004509 // Trigger quality adapt up, expect upscaled resolution (480x270).
4510 video_stream_encoder_->TriggerQualityHigh();
4511 timestamp_ms += kFrameIntervalMs;
4512 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4513 WaitForEncodedFrame(timestamp_ms);
4514 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4515 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4516 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4517 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4518 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4519
4520 // Trigger quality and cpu adapt up since both are most limited, expect
4521 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004522 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004523 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004524 timestamp_ms += kFrameIntervalMs;
4525 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4526 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004527 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004528 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4529 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4530 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004531 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004532
Evan Shrubsole64469032020-06-11 10:45:29 +02004533 // Trigger quality and cpu adapt up since both are most limited, expect
4534 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004535 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004536 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004537 timestamp_ms += kFrameIntervalMs;
4538 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4539 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004540 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004541 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004542 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004543 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004544 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4545 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004546
Evan Shrubsole64469032020-06-11 10:45:29 +02004547 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4548 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004549 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004550 timestamp_ms += kFrameIntervalMs;
4551 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4552 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004553 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004554 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4555 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004556 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004557 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004558
4559 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004560 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004561 timestamp_ms += kFrameIntervalMs;
4562 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004563 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004564 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004565 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004566 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4567 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004568 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004569 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004570
mflodmancc3d4422017-08-03 08:27:51 -07004571 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004572}
4573
mflodmancc3d4422017-08-03 08:27:51 -07004574TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004575 const int kWidth = 640;
4576 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004577
Henrik Boström381d1092020-05-12 18:49:07 +02004578 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004579 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004580
perkj803d97f2016-11-01 11:45:46 -07004581 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004582 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004583 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004584 }
4585
mflodmancc3d4422017-08-03 08:27:51 -07004586 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004587 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004588 video_source_.IncomingCapturedFrame(CreateFrame(
4589 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004590 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004591 }
4592
mflodmancc3d4422017-08-03 08:27:51 -07004593 video_stream_encoder_->Stop();
4594 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004595 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004596
Ying Wangef3998f2019-12-09 13:06:53 +01004597 EXPECT_METRIC_EQ(
4598 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4599 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004600 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4601}
4602
mflodmancc3d4422017-08-03 08:27:51 -07004603TEST_F(VideoStreamEncoderTest,
4604 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004605 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004606 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004607 const int kWidth = 640;
4608 const int kHeight = 360;
4609
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004610 video_stream_encoder_->SetSource(&video_source_,
4611 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004612
4613 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4614 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004615 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004616 }
4617
mflodmancc3d4422017-08-03 08:27:51 -07004618 video_stream_encoder_->Stop();
4619 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004620 stats_proxy_.reset();
4621
4622 EXPECT_EQ(0,
4623 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4624}
4625
Per Kjellanderdcef6412020-10-07 15:09:05 +02004626TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4627 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004628 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004629 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004630
4631 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004632 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 10:07:11 +02004633 SimulcastRateAllocator(fake_encoder_.config())
4634 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004635 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004636
Henrik Boström381d1092020-05-12 18:49:07 +02004637 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004638 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004639
sprang57c2fff2017-01-16 06:24:02 -08004640 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004641 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4642 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004643 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4644 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4645
Erik Språngd7329ca2019-02-21 21:19:53 +01004646 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004647 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004648 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004649
Per Kjellanderdcef6412020-10-07 15:09:05 +02004650 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004651 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004652 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4653 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004654 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004655 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004656
Per Kjellanderdcef6412020-10-07 15:09:05 +02004657 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004658 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004659 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004660 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004661 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4662 WaitForEncodedFrame(CurrentTimeMs());
4663 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004664 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004665 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004666
mflodmancc3d4422017-08-03 08:27:51 -07004667 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004668}
4669
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004670TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004671 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004672 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004673 kVideoLayersAllocation);
4674
4675 const int kDefaultFps = 30;
4676
4677 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004678 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004679
4680 video_source_.IncomingCapturedFrame(
4681 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4682 WaitForEncodedFrame(CurrentTimeMs());
4683 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4684 VideoLayersAllocation last_layer_allocation =
4685 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004686 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004687 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4688
4689 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004690 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004691 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 10:07:11 +02004692 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 17:53:22 +02004693 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4694
Erik Språng9d69cbe2020-10-22 17:44:42 +02004695 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004696 int number_of_layers_allocation = 1;
4697 const int64_t start_time_ms = CurrentTimeMs();
4698 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4699 video_source_.IncomingCapturedFrame(
4700 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4701 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004702 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4703 number_of_layers_allocation = sink_.number_of_layers_allocations();
4704 VideoLayersAllocation new_allocation =
4705 sink_.GetLastVideoLayersAllocation();
4706 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4707 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4708 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4709 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4710 .target_bitrate_per_temporal_layer,
4711 last_layer_allocation.active_spatial_layers[0]
4712 .target_bitrate_per_temporal_layer);
4713 last_layer_allocation = new_allocation;
4714 }
4715 }
4716 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4717 video_stream_encoder_->Stop();
4718}
4719
4720TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004721 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004722 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4723 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4724 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004725 VideoEncoderConfig video_encoder_config;
4726 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4727 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004728 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004729 video_encoder_config.content_type =
4730 VideoEncoderConfig::ContentType::kRealtimeVideo;
4731 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004732 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004733 VideoEncoder::GetDefaultVp8Settings());
4734 for (auto& layer : video_encoder_config.simulcast_layers) {
4735 layer.num_temporal_layers = 2;
4736 }
4737 // Simulcast layers are used for enabling/disabling streams.
4738 video_encoder_config.simulcast_layers[0].active = true;
4739 video_encoder_config.simulcast_layers[1].active = false;
4740 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004741 ConfigureEncoder(std::move(video_encoder_config),
4742 VideoStreamEncoder::BitrateAllocationCallbackType::
4743 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004744
4745 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004746 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004747
4748 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4749 WaitForEncodedFrame(CurrentTimeMs());
4750 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4751 VideoLayersAllocation last_layer_allocation =
4752 sink_.GetLastVideoLayersAllocation();
4753
4754 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4755 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4756 .target_bitrate_per_temporal_layer,
4757 SizeIs(2));
4758 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4759 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4760 video_stream_encoder_->Stop();
4761}
4762
4763TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004764 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004765 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4766 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4767 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004768 VideoEncoderConfig video_encoder_config;
4769 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4770 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004771 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004772 video_encoder_config.content_type =
4773 VideoEncoderConfig::ContentType::kRealtimeVideo;
4774 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004775 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004776 VideoEncoder::GetDefaultVp8Settings());
4777 for (auto& layer : video_encoder_config.simulcast_layers) {
4778 layer.num_temporal_layers = 2;
4779 }
4780 // Simulcast layers are used for enabling/disabling streams.
4781 video_encoder_config.simulcast_layers[0].active = true;
4782 video_encoder_config.simulcast_layers[1].active = false;
4783 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004784 ConfigureEncoder(std::move(video_encoder_config),
4785 VideoStreamEncoder::BitrateAllocationCallbackType::
4786 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004787
4788 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004789 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004790
4791 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4792 WaitForEncodedFrame(CurrentTimeMs());
4793 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4794 VideoLayersAllocation last_layer_allocation =
4795 sink_.GetLastVideoLayersAllocation();
4796
4797 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4798 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4799 .target_bitrate_per_temporal_layer,
4800 SizeIs(2));
4801 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4802
4803 video_stream_encoder_->Stop();
4804}
4805
4806TEST_F(VideoStreamEncoderTest,
4807 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4808 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4809 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004810 VideoEncoderConfig video_encoder_config;
4811 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4812 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004813 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004814 video_encoder_config.content_type =
4815 VideoEncoderConfig::ContentType::kRealtimeVideo;
4816 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4817 vp9_settings.numberOfSpatialLayers = 2;
4818 vp9_settings.numberOfTemporalLayers = 2;
4819 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4820 vp9_settings.automaticResizeOn = false;
4821 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004822 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004823 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004824 ConfigureEncoder(std::move(video_encoder_config),
4825 VideoStreamEncoder::BitrateAllocationCallbackType::
4826 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004827
4828 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004829 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004830
4831 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4832 WaitForEncodedFrame(CurrentTimeMs());
4833 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4834 VideoLayersAllocation last_layer_allocation =
4835 sink_.GetLastVideoLayersAllocation();
4836
4837 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4838 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4839 .target_bitrate_per_temporal_layer,
4840 SizeIs(2));
4841 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4842 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4843 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4844 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4845 .target_bitrate_per_temporal_layer,
4846 SizeIs(2));
4847 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4848 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4849 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4850
4851 // Since full SVC is used, expect the top layer to utilize the full target
4852 // rate.
4853 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4854 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004855 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004856 video_stream_encoder_->Stop();
4857}
4858
4859TEST_F(VideoStreamEncoderTest,
4860 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4861 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4862 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004863 VideoEncoderConfig video_encoder_config;
4864 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4865 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004866 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004867 video_encoder_config.content_type =
4868 VideoEncoderConfig::ContentType::kRealtimeVideo;
4869 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4870 vp9_settings.numberOfSpatialLayers = 2;
4871 vp9_settings.numberOfTemporalLayers = 2;
4872 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4873 vp9_settings.automaticResizeOn = false;
4874 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004875 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004876 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004877 ConfigureEncoder(std::move(video_encoder_config),
4878 VideoStreamEncoder::BitrateAllocationCallbackType::
4879 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004880
4881 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004882 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004883
4884 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4885 WaitForEncodedFrame(CurrentTimeMs());
4886 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4887 VideoLayersAllocation last_layer_allocation =
4888 sink_.GetLastVideoLayersAllocation();
4889
4890 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4891 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4892 .target_bitrate_per_temporal_layer,
4893 SizeIs(1));
4894 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4895 .target_bitrate_per_temporal_layer,
4896 SizeIs(1));
4897 // Since full SVC is used, expect the top layer to utilize the full target
4898 // rate.
4899 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4900 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004901 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004902 video_stream_encoder_->Stop();
4903}
4904
4905TEST_F(VideoStreamEncoderTest,
4906 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4907 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4908 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004909 VideoEncoderConfig video_encoder_config;
4910 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4911 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004912 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004913 video_encoder_config.content_type =
4914 VideoEncoderConfig::ContentType::kRealtimeVideo;
4915 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4916 vp9_settings.numberOfSpatialLayers = 2;
4917 vp9_settings.numberOfTemporalLayers = 2;
4918 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4919 vp9_settings.automaticResizeOn = false;
4920 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004921 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004922 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004923 ConfigureEncoder(std::move(video_encoder_config),
4924 VideoStreamEncoder::BitrateAllocationCallbackType::
4925 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004926
4927 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004928 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004929
4930 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4931 WaitForEncodedFrame(CurrentTimeMs());
4932 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4933 VideoLayersAllocation last_layer_allocation =
4934 sink_.GetLastVideoLayersAllocation();
4935
4936 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4937 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4938 .target_bitrate_per_temporal_layer,
4939 SizeIs(2));
4940 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4941 .target_bitrate_per_temporal_layer,
4942 SizeIs(2));
4943 // Since KSVC is, spatial layers are independend except on key frames.
4944 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4945 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004946 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004947 video_stream_encoder_->Stop();
4948}
4949
4950TEST_F(VideoStreamEncoderTest,
4951 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4952 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4953 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4954 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004955 VideoEncoderConfig video_encoder_config;
4956 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4957 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004958 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004959 video_encoder_config.content_type =
4960 VideoEncoderConfig::ContentType::kRealtimeVideo;
4961 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4962 vp9_settings.numberOfSpatialLayers = 3;
4963 vp9_settings.numberOfTemporalLayers = 2;
4964 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4965 vp9_settings.automaticResizeOn = false;
4966 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004967 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004968 vp9_settings);
4969 // Simulcast layers are used for enabling/disabling streams.
4970 video_encoder_config.simulcast_layers.resize(3);
4971 video_encoder_config.simulcast_layers[0].active = false;
4972 video_encoder_config.simulcast_layers[1].active = true;
4973 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004974 ConfigureEncoder(std::move(video_encoder_config),
4975 VideoStreamEncoder::BitrateAllocationCallbackType::
4976 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004977
4978 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004979 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004980
4981 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4982 WaitForEncodedFrame(CurrentTimeMs());
4983 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4984 VideoLayersAllocation last_layer_allocation =
4985 sink_.GetLastVideoLayersAllocation();
4986
4987 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4988 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4989 .target_bitrate_per_temporal_layer,
4990 SizeIs(2));
4991 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4992 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4993
4994 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4995 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4996 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4997 .target_bitrate_per_temporal_layer,
4998 SizeIs(2));
4999 // Since full SVC is used, expect the top layer to utilize the full target
5000 // rate.
5001 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
5002 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02005003 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005004 video_stream_encoder_->Stop();
5005}
5006
5007TEST_F(VideoStreamEncoderTest,
5008 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
5009 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5010 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5011 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005012 VideoEncoderConfig video_encoder_config;
5013 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5014 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005015 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005016 video_encoder_config.content_type =
5017 VideoEncoderConfig::ContentType::kRealtimeVideo;
5018 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5019 vp9_settings.numberOfSpatialLayers = 3;
5020 vp9_settings.numberOfTemporalLayers = 2;
5021 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5022 vp9_settings.automaticResizeOn = false;
5023 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005024 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005025 vp9_settings);
5026 // Simulcast layers are used for enabling/disabling streams.
5027 video_encoder_config.simulcast_layers.resize(3);
5028 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005029 ConfigureEncoder(std::move(video_encoder_config),
5030 VideoStreamEncoder::BitrateAllocationCallbackType::
5031 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005032
5033 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005034 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005035
5036 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5037 WaitForEncodedFrame(CurrentTimeMs());
5038 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5039 VideoLayersAllocation last_layer_allocation =
5040 sink_.GetLastVideoLayersAllocation();
5041
5042 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
5043 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5044 .target_bitrate_per_temporal_layer,
5045 SizeIs(2));
5046 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
5047 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5048
5049 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
5050 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
5051 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
5052 .target_bitrate_per_temporal_layer,
5053 SizeIs(2));
5054 video_stream_encoder_->Stop();
5055}
5056
5057TEST_F(VideoStreamEncoderTest,
5058 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
5059 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5060 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5061 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005062 VideoEncoderConfig video_encoder_config;
5063 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5064 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005065 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005066 video_encoder_config.content_type =
5067 VideoEncoderConfig::ContentType::kRealtimeVideo;
5068 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5069 vp9_settings.numberOfSpatialLayers = 3;
5070 vp9_settings.numberOfTemporalLayers = 2;
5071 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5072 vp9_settings.automaticResizeOn = false;
5073 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005074 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005075 vp9_settings);
5076 // Simulcast layers are used for enabling/disabling streams.
5077 video_encoder_config.simulcast_layers.resize(3);
5078 video_encoder_config.simulcast_layers[0].active = false;
5079 video_encoder_config.simulcast_layers[1].active = false;
5080 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005081 ConfigureEncoder(std::move(video_encoder_config),
5082 VideoStreamEncoder::BitrateAllocationCallbackType::
5083 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005084
5085 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005086 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005087
5088 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5089 WaitForEncodedFrame(CurrentTimeMs());
5090 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5091 VideoLayersAllocation last_layer_allocation =
5092 sink_.GetLastVideoLayersAllocation();
5093
5094 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5095 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5096 .target_bitrate_per_temporal_layer,
5097 SizeIs(2));
5098 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5099 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5100 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5101 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02005102 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005103 video_stream_encoder_->Stop();
5104}
5105
5106TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5107 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005108 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005109 kVideoLayersAllocation);
5110 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005111 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005112
5113 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5114 WaitForEncodedFrame(CurrentTimeMs());
5115 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5116 VideoLayersAllocation last_layer_allocation =
5117 sink_.GetLastVideoLayersAllocation();
5118
5119 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5120 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5121 .target_bitrate_per_temporal_layer,
5122 SizeIs(1));
5123 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5124 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005125 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005126 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5127 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5128 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5129 video_stream_encoder_->Stop();
5130}
5131
5132TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02005133 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5134 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005135 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02005136 kVideoLayersAllocation);
5137
5138 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005139 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005140
5141 video_source_.IncomingCapturedFrame(
5142 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5143 WaitForEncodedFrame(CurrentTimeMs());
5144 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5145 VideoLayersAllocation last_layer_allocation =
5146 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02005147 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02005148 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5149 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5150 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005151 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 17:53:22 +02005152
5153 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005154 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5155 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005156 video_source_.IncomingCapturedFrame(
5157 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5158 WaitForEncodedFrame(CurrentTimeMs());
5159
5160 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5161 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5162 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5163 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5164 .target_bitrate_per_temporal_layer[0],
5165 DataRate::Zero());
5166
5167 video_stream_encoder_->Stop();
5168}
5169
Per Kjellander4190ce92020-12-15 17:24:55 +01005170TEST_F(VideoStreamEncoderTest,
5171 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5172 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005173 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01005174 kVideoLayersAllocation);
5175
5176 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005177 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5178 0, 0, 0);
Per Kjellander4190ce92020-12-15 17:24:55 +01005179
5180 video_source_.IncomingCapturedFrame(
5181 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5182 WaitForEncodedFrame(CurrentTimeMs());
5183 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5184 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5185 SizeIs(2));
5186 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5187 codec_width_);
5188 EXPECT_EQ(
5189 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5190 codec_height_);
5191
5192 video_source_.IncomingCapturedFrame(
5193 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5194 WaitForEncodedFrame(CurrentTimeMs());
5195 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5196 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5197 SizeIs(2));
5198 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5199 codec_width_ / 2);
5200 EXPECT_EQ(
5201 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5202 codec_height_ / 2);
5203
5204 video_stream_encoder_->Stop();
5205}
5206
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005207TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5208 // 2 TLs configured, temporal layers supported by encoder.
5209 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005210 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005211 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005212 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005213 fake_encoder_.SetTemporalLayersSupported(0, true);
5214
5215 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 10:07:11 +02005216 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005217 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005218 kNumTemporalLayers, /*temporal_id*/ 0,
5219 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005220 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005221 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005222 kNumTemporalLayers, /*temporal_id*/ 1,
5223 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005224 VideoBitrateAllocation expected_bitrate;
5225 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5226 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5227
5228 VerifyAllocatedBitrate(expected_bitrate);
5229 video_stream_encoder_->Stop();
5230}
5231
5232TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5233 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005234 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005235 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005236 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005237 fake_encoder_.SetTemporalLayersSupported(0, false);
5238
5239 // Temporal layers not supported by the encoder.
5240 // Total bitrate should be at ti:0.
5241 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005242 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005243
5244 VerifyAllocatedBitrate(expected_bitrate);
5245 video_stream_encoder_->Stop();
5246}
5247
5248TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005249 webrtc::test::ScopedKeyValueConfig field_trials(
5250 field_trials_,
Per Kjellanderdcef6412020-10-07 15:09:05 +02005251 "WebRTC-Video-QualityScalerSettings/"
5252 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5253 // Reset encoder for field trials to take effect.
5254 ConfigureEncoder(video_encoder_config_.Copy());
5255
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005256 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005257 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005258 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005259 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005260 fake_encoder_.SetTemporalLayersSupported(0, true);
5261 fake_encoder_.SetTemporalLayersSupported(1, false);
5262
5263 const int kS0Bps = 150000;
5264 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005265 kS0Bps *
5266 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5267 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005268 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005269 kS0Bps *
5270 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5271 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005272 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005273 // Temporal layers not supported by si:1.
5274 VideoBitrateAllocation expected_bitrate;
5275 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5276 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5277 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5278
5279 VerifyAllocatedBitrate(expected_bitrate);
5280 video_stream_encoder_->Stop();
5281}
5282
Niels Möller7dc26b72017-12-06 10:27:48 +01005283TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5284 const int kFrameWidth = 1280;
5285 const int kFrameHeight = 720;
5286 const int kFramerate = 24;
5287
Henrik Boström381d1092020-05-12 18:49:07 +02005288 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005289 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005290 test::FrameForwarder source;
5291 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005292 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005293
5294 // Insert a single frame, triggering initial configuration.
5295 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5296 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5297
5298 EXPECT_EQ(
5299 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5300 kDefaultFramerate);
5301
5302 // Trigger reconfigure encoder (without resetting the entire instance).
5303 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005304 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5305 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005306 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005307 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005308 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005309 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5310
5311 // Detector should be updated with fps limit from codec config.
5312 EXPECT_EQ(
5313 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5314 kFramerate);
5315
5316 // Trigger overuse, max framerate should be reduced.
5317 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5318 stats.input_frame_rate = kFramerate;
5319 stats_proxy_->SetMockStats(stats);
5320 video_stream_encoder_->TriggerCpuOveruse();
5321 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5322 int adapted_framerate =
5323 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5324 EXPECT_LT(adapted_framerate, kFramerate);
5325
5326 // Trigger underuse, max framerate should go back to codec configured fps.
5327 // Set extra low fps, to make sure it's actually reset, not just incremented.
5328 stats = stats_proxy_->GetStats();
5329 stats.input_frame_rate = adapted_framerate / 2;
5330 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005331 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005332 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5333 EXPECT_EQ(
5334 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5335 kFramerate);
5336
5337 video_stream_encoder_->Stop();
5338}
5339
5340TEST_F(VideoStreamEncoderTest,
5341 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5342 const int kFrameWidth = 1280;
5343 const int kFrameHeight = 720;
5344 const int kLowFramerate = 15;
5345 const int kHighFramerate = 25;
5346
Henrik Boström381d1092020-05-12 18:49:07 +02005347 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005348 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005349 test::FrameForwarder source;
5350 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005351 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005352
5353 // Trigger initial configuration.
5354 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005355 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5356 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005357 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005358 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005359 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005360 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005361 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5362
5363 EXPECT_EQ(
5364 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5365 kLowFramerate);
5366
5367 // Trigger overuse, max framerate should be reduced.
5368 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5369 stats.input_frame_rate = kLowFramerate;
5370 stats_proxy_->SetMockStats(stats);
5371 video_stream_encoder_->TriggerCpuOveruse();
5372 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5373 int adapted_framerate =
5374 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5375 EXPECT_LT(adapted_framerate, kLowFramerate);
5376
5377 // Reconfigure the encoder with a new (higher max framerate), max fps should
5378 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005379 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005380 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5381 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005382 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005383 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5384
5385 EXPECT_EQ(
5386 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5387 adapted_framerate);
5388
5389 // Trigger underuse, max framerate should go back to codec configured fps.
5390 stats = stats_proxy_->GetStats();
5391 stats.input_frame_rate = adapted_framerate;
5392 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005393 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005394 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5395 EXPECT_EQ(
5396 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5397 kHighFramerate);
5398
5399 video_stream_encoder_->Stop();
5400}
5401
mflodmancc3d4422017-08-03 08:27:51 -07005402TEST_F(VideoStreamEncoderTest,
5403 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005404 const int kFrameWidth = 1280;
5405 const int kFrameHeight = 720;
5406 const int kFramerate = 24;
5407
Henrik Boström381d1092020-05-12 18:49:07 +02005408 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005409 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005410 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005411 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005412 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005413
5414 // Trigger initial configuration.
5415 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005416 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5417 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005418 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 04:21:07 -07005419 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005420 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005421 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005422 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005423
Niels Möller7dc26b72017-12-06 10:27:48 +01005424 EXPECT_EQ(
5425 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5426 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005427
5428 // Trigger overuse, max framerate should be reduced.
5429 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5430 stats.input_frame_rate = kFramerate;
5431 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005432 video_stream_encoder_->TriggerCpuOveruse();
5433 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005434 int adapted_framerate =
5435 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005436 EXPECT_LT(adapted_framerate, kFramerate);
5437
5438 // Change degradation preference to not enable framerate scaling. Target
5439 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005440 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005441 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005442 EXPECT_EQ(
5443 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5444 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005445
mflodmancc3d4422017-08-03 08:27:51 -07005446 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005447}
5448
mflodmancc3d4422017-08-03 08:27:51 -07005449TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005450 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005451 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005452 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5453 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5454 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005455 const int kWidth = 640;
5456 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005457
asaperssonfab67072017-04-04 05:51:49 -07005458 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005459
5460 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005461 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005462
5463 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005464 EXPECT_TRUE_WAIT(
5465 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005466
sprangc5d62e22017-04-02 23:53:04 -07005467 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005468
asaperssonfab67072017-04-04 05:51:49 -07005469 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005470 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005471 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005472
5473 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005474 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005475
Henrik Boström2671dac2020-05-19 16:29:09 +02005476 EXPECT_TRUE_WAIT(
5477 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005478
mflodmancc3d4422017-08-03 08:27:51 -07005479 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005480}
5481
mflodmancc3d4422017-08-03 08:27:51 -07005482TEST_F(VideoStreamEncoderTest,
5483 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005484 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005485 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005486 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5487 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5488 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005489 const int kWidth = 640;
5490 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005491
5492 // We expect the n initial frames to get dropped.
5493 int i;
5494 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005495 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005496 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005497 }
5498 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005499 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005500 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005501
5502 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005503 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005504
mflodmancc3d4422017-08-03 08:27:51 -07005505 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005506}
5507
mflodmancc3d4422017-08-03 08:27:51 -07005508TEST_F(VideoStreamEncoderTest,
5509 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005510 const int kWidth = 640;
5511 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005512 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005513 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005514
5515 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005516 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005517 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005518
asaperssonfab67072017-04-04 05:51:49 -07005519 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005520 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005521 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005522
mflodmancc3d4422017-08-03 08:27:51 -07005523 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005524}
5525
mflodmancc3d4422017-08-03 08:27:51 -07005526TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005527 const int kWidth = 640;
5528 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005529 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005530
5531 VideoEncoderConfig video_encoder_config;
5532 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5533 // Make format different, to force recreation of encoder.
5534 video_encoder_config.video_format.parameters["foo"] = "foo";
5535 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005536 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005537 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005538 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005539
kthelgasonb83797b2017-02-14 11:57:25 -08005540 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005541 video_stream_encoder_->SetSource(&video_source_,
5542 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005543
asaperssonfab67072017-04-04 05:51:49 -07005544 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005545 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005546 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005547
mflodmancc3d4422017-08-03 08:27:51 -07005548 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005549 fake_encoder_.SetQualityScaling(true);
5550}
5551
Åsa Persson139f4dc2019-08-02 09:29:58 +02005552TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005553 webrtc::test::ScopedKeyValueConfig field_trials(
5554 field_trials_,
Åsa Persson139f4dc2019-08-02 09:29:58 +02005555 "WebRTC-Video-QualityScalerSettings/"
5556 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5557 // Reset encoder for field trials to take effect.
5558 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005559 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5560 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 09:29:58 +02005561 const int kWidth = 640;
5562 const int kHeight = 360;
5563
Henrik Boström381d1092020-05-12 18:49:07 +02005564 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005565 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005566 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5567 // Frame should not be dropped.
5568 WaitForEncodedFrame(1);
5569
Henrik Boström381d1092020-05-12 18:49:07 +02005570 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005571 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5572 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5573 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005574 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5575 // Frame should not be dropped.
5576 WaitForEncodedFrame(2);
5577
Henrik Boström381d1092020-05-12 18:49:07 +02005578 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005579 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5580 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5581 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005582 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5583 // Expect to drop this frame, the wait should time out.
5584 ExpectDroppedFrame();
5585
5586 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005587 EXPECT_TRUE_WAIT(
5588 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005589 video_stream_encoder_->Stop();
5590}
5591
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005592TEST_F(VideoStreamEncoderTest,
5593 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005594 webrtc::test::ScopedKeyValueConfig field_trials(
5595 field_trials_,
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005596 "WebRTC-Video-QualityScalerSettings/"
5597 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5598 fake_encoder_.SetQualityScaling(false);
5599 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005600 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5601 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005602 const int kWidth = 640;
5603 const int kHeight = 360;
5604
5605 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005606 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005607 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5608 // Frame should not be dropped.
5609 WaitForEncodedFrame(1);
5610
5611 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5612 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5613 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5614 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5615 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5616 // Frame should not be dropped.
5617 WaitForEncodedFrame(2);
5618
5619 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5620 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5621 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5622 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5623 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5624 // Not dropped since quality scaling is disabled.
5625 WaitForEncodedFrame(3);
5626
5627 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005628 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005629 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5630
5631 video_stream_encoder_->Stop();
5632}
5633
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005634TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005635 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005636 // Set simulcast.
5637 ResetEncoder("VP8", 3, 1, 1, false);
5638 fake_encoder_.SetQualityScaling(true);
5639 const int kWidth = 1280;
5640 const int kHeight = 720;
5641 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005642 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005643 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5644 // Frame should not be dropped.
5645 WaitForEncodedFrame(1);
5646
5647 // Trigger QVGA "singlecast"
5648 // Update the config.
5649 VideoEncoderConfig video_encoder_config;
5650 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5651 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005652 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005653 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005654 "VP8", /*max qp*/ 56, /*screencast*/ false,
5655 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005656 for (auto& layer : video_encoder_config.simulcast_layers) {
5657 layer.num_temporal_layers = 1;
5658 layer.max_framerate = kDefaultFramerate;
5659 }
Asa Persson606d3cb2021-10-04 10:07:11 +02005660 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005661 video_encoder_config.content_type =
5662 VideoEncoderConfig::ContentType::kRealtimeVideo;
5663
5664 video_encoder_config.simulcast_layers[0].active = true;
5665 video_encoder_config.simulcast_layers[1].active = false;
5666 video_encoder_config.simulcast_layers[2].active = false;
5667
5668 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5669 kMaxPayloadLength);
5670 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5671
5672 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5673 // Frame should not be dropped.
5674 WaitForEncodedFrame(2);
5675
5676 // Trigger HD "singlecast"
5677 video_encoder_config.simulcast_layers[0].active = false;
5678 video_encoder_config.simulcast_layers[1].active = false;
5679 video_encoder_config.simulcast_layers[2].active = true;
5680
5681 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5682 kMaxPayloadLength);
5683 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5684
5685 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5686 // Frame should be dropped because of initial frame drop.
5687 ExpectDroppedFrame();
5688
5689 // Expect the sink_wants to specify a scaled frame.
5690 EXPECT_TRUE_WAIT(
5691 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5692 video_stream_encoder_->Stop();
5693}
5694
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005695TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005696 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005697 // Set simulcast.
5698 ResetEncoder("VP9", 1, 1, 3, false);
5699 fake_encoder_.SetQualityScaling(true);
5700 const int kWidth = 1280;
5701 const int kHeight = 720;
5702 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005703 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005704 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5705 // Frame should not be dropped.
5706 WaitForEncodedFrame(1);
5707
5708 // Trigger QVGA "singlecast"
5709 // Update the config.
5710 VideoEncoderConfig video_encoder_config;
5711 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5712 &video_encoder_config);
5713 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5714 vp9_settings.numberOfSpatialLayers = 3;
5715 // Since only one layer is active - automatic resize should be enabled.
5716 vp9_settings.automaticResizeOn = true;
5717 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005718 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005719 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005720 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005721 video_encoder_config.content_type =
5722 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 16:22:11 +02005723 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005724 // which SVC layers are active.
5725 video_encoder_config.simulcast_layers.resize(3);
5726
5727 video_encoder_config.simulcast_layers[0].active = true;
5728 video_encoder_config.simulcast_layers[1].active = false;
5729 video_encoder_config.simulcast_layers[2].active = false;
5730
5731 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5732 kMaxPayloadLength);
5733 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5734
5735 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5736 // Frame should not be dropped.
5737 WaitForEncodedFrame(2);
5738
5739 // Trigger HD "singlecast"
5740 video_encoder_config.simulcast_layers[0].active = false;
5741 video_encoder_config.simulcast_layers[1].active = false;
5742 video_encoder_config.simulcast_layers[2].active = true;
5743
5744 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5745 kMaxPayloadLength);
5746 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5747
5748 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5749 // Frame should be dropped because of initial frame drop.
5750 ExpectDroppedFrame();
5751
5752 // Expect the sink_wants to specify a scaled frame.
5753 EXPECT_TRUE_WAIT(
5754 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5755 video_stream_encoder_->Stop();
5756}
5757
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005758TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005759 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5760 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5761 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5762 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5763 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5764 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5765 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5766 fake_encoder_.SetResolutionBitrateLimits(
5767 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5768
5769 VideoEncoderConfig video_encoder_config;
5770 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5771 &video_encoder_config);
5772 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5773 vp9_settings.numberOfSpatialLayers = 3;
5774 // Since only one layer is active - automatic resize should be enabled.
5775 vp9_settings.automaticResizeOn = true;
5776 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005777 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005778 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005779 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005780 video_encoder_config.content_type =
5781 VideoEncoderConfig::ContentType::kRealtimeVideo;
5782 // Simulcast layers are used to indicate which spatial layers are active.
5783 video_encoder_config.simulcast_layers.resize(3);
5784 video_encoder_config.simulcast_layers[0].active = false;
5785 video_encoder_config.simulcast_layers[1].active = true;
5786 video_encoder_config.simulcast_layers[2].active = false;
5787
5788 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5789 kMaxPayloadLength);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005790
5791 // The encoder bitrate limits for 360p should be used.
5792 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005793 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005794 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5795 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5796 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5797 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5798 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5799 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005800 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005801 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005802 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005803 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005804
5805 // The encoder bitrate limits for 270p should be used.
5806 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02005807 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005808 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5809 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5810 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5811 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5812 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5813 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005814 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005815 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005816 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005817 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005818
5819 video_stream_encoder_->Stop();
5820}
5821
5822TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005823 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5824 VideoEncoderConfig video_encoder_config;
5825 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5826 &video_encoder_config);
5827 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5828 vp9_settings.numberOfSpatialLayers = 3;
5829 // Since only one layer is active - automatic resize should be enabled.
5830 vp9_settings.automaticResizeOn = true;
5831 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005832 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005833 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005834 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005835 video_encoder_config.content_type =
5836 VideoEncoderConfig::ContentType::kRealtimeVideo;
5837 // Simulcast layers are used to indicate which spatial layers are active.
5838 video_encoder_config.simulcast_layers.resize(3);
5839 video_encoder_config.simulcast_layers[0].active = false;
5840 video_encoder_config.simulcast_layers[1].active = true;
5841 video_encoder_config.simulcast_layers[2].active = false;
5842
5843 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5844 kMaxPayloadLength);
Åsa Persson258e9892021-02-25 10:39:51 +01005845
5846 // The default bitrate limits for 360p should be used.
5847 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005848 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5849 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005850 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005851 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005852 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5853 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5854 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5855 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5856 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5857 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005858 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005859 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005860 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005861 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005862
5863 // The default bitrate limits for 270p should be used.
5864 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005865 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5866 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005867 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02005868 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005869 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5870 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5871 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5872 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5873 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5874 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005875 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005876 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005877 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005878 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005879
5880 video_stream_encoder_->Stop();
5881}
5882
5883TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005884 webrtc::test::ScopedKeyValueConfig field_trials(
5885 field_trials_, "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
Åsa Persson258e9892021-02-25 10:39:51 +01005886 VideoEncoderConfig video_encoder_config;
5887 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5888 &video_encoder_config);
5889 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5890 vp9_settings.numberOfSpatialLayers = 3;
5891 // Since only one layer is active - automatic resize should be enabled.
5892 vp9_settings.automaticResizeOn = true;
5893 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005894 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005895 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005896 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005897 video_encoder_config.content_type =
5898 VideoEncoderConfig::ContentType::kRealtimeVideo;
5899 // Simulcast layers are used to indicate which spatial layers are active.
5900 video_encoder_config.simulcast_layers.resize(3);
5901 video_encoder_config.simulcast_layers[0].active = false;
5902 video_encoder_config.simulcast_layers[1].active = true;
5903 video_encoder_config.simulcast_layers[2].active = false;
5904
5905 // Reset encoder for field trials to take effect.
5906 ConfigureEncoder(video_encoder_config.Copy());
5907
5908 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5909 kMaxPayloadLength);
Åsa Persson258e9892021-02-25 10:39:51 +01005910
5911 // The default bitrate limits for 360p should not be used.
5912 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005913 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5914 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005915 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005916 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005917 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5918 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5919 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5920 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5921 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5922 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005923 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005924 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005925
5926 video_stream_encoder_->Stop();
5927}
5928
5929TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5930 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5931 /*num_spatial_layers=*/1, /*screenshare=*/false);
5932
5933 // The default singlecast bitrate limits for 720p should not be used.
5934 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005935 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5936 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005937 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005938 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005939 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5940 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5941 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5942 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5943 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5944 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005945 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005946 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005947
5948 video_stream_encoder_->Stop();
5949}
5950
5951TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005952 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5953 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5954 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5955 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5956 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5957 fake_encoder_.SetResolutionBitrateLimits(
5958 {kEncoderLimits180p, kEncoderLimits720p});
5959
5960 VideoEncoderConfig video_encoder_config;
5961 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5962 &video_encoder_config);
5963 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5964 vp9_settings.numberOfSpatialLayers = 3;
5965 // Since only one layer is active - automatic resize should be enabled.
5966 vp9_settings.automaticResizeOn = true;
5967 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005968 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005969 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005970 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005971 video_encoder_config.content_type =
5972 VideoEncoderConfig::ContentType::kRealtimeVideo;
5973 // Simulcast layers are used to indicate which spatial layers are active.
5974 video_encoder_config.simulcast_layers.resize(3);
5975 video_encoder_config.simulcast_layers[0].active = true;
5976 video_encoder_config.simulcast_layers[1].active = false;
5977 video_encoder_config.simulcast_layers[2].active = false;
5978
5979 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5980 kMaxPayloadLength);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005981
5982 // Limits not applied on lowest stream, limits for 180p should not be used.
5983 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005984 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005985 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5986 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5987 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5988 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5989 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5990 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005991 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005992 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005993 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005994 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005995
5996 video_stream_encoder_->Stop();
5997}
5998
5999TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006000 InitialFrameDropActivatesWhenResolutionIncreases) {
6001 const int kWidth = 640;
6002 const int kHeight = 360;
6003
6004 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006005 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006006 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
6007 // Frame should not be dropped.
6008 WaitForEncodedFrame(1);
6009
6010 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006011 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006012 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
6013 // Frame should not be dropped, bitrate not too low for frame.
6014 WaitForEncodedFrame(2);
6015
6016 // Incoming resolution increases.
6017 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
6018 // Expect to drop this frame, bitrate too low for frame.
6019 ExpectDroppedFrame();
6020
6021 // Expect the sink_wants to specify a scaled frame.
6022 EXPECT_TRUE_WAIT(
6023 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
6024 video_stream_encoder_->Stop();
6025}
6026
6027TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
6028 const int kWidth = 640;
6029 const int kHeight = 360;
6030 // So that quality scaling doesn't happen by itself.
6031 fake_encoder_.SetQp(kQpHigh);
6032
6033 AdaptingFrameForwarder source(&time_controller_);
6034 source.set_adaptation_enabled(true);
6035 video_stream_encoder_->SetSource(
6036 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
6037
6038 int timestamp = 1;
6039
6040 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006041 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006042 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6043 WaitForEncodedFrame(timestamp);
6044 timestamp += 9000;
6045 // Long pause to disable all first BWE drop logic.
6046 AdvanceTime(TimeDelta::Millis(1000));
6047
6048 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006049 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006050 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6051 // Not dropped frame, as initial frame drop is disabled by now.
6052 WaitForEncodedFrame(timestamp);
6053 timestamp += 9000;
6054 AdvanceTime(TimeDelta::Millis(100));
6055
6056 // Quality adaptation down.
6057 video_stream_encoder_->TriggerQualityLow();
6058
6059 // Adaptation has an effect.
6060 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6061 5000);
6062
6063 // Frame isn't dropped as initial frame dropper is disabled.
6064 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6065 WaitForEncodedFrame(timestamp);
6066 timestamp += 9000;
6067 AdvanceTime(TimeDelta::Millis(100));
6068
6069 // Quality adaptation up.
6070 video_stream_encoder_->TriggerQualityHigh();
6071
6072 // Adaptation has an effect.
6073 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
6074 5000);
6075
6076 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6077 // Frame should not be dropped, as initial framedropper is off.
6078 WaitForEncodedFrame(timestamp);
6079
6080 video_stream_encoder_->Stop();
6081}
6082
Åsa Persson7f354f82021-02-04 15:52:15 +01006083TEST_F(VideoStreamEncoderTest,
6084 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
6085 const int kMinStartBps360p = 222000;
6086 fake_encoder_.SetResolutionBitrateLimits(
6087 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6088 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6089 800000)});
6090
6091 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6092 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6093 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6094 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
6095 0, 0, 0);
6096 // Frame should not be dropped, bitrate not too low for frame.
6097 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6098 WaitForEncodedFrame(1);
6099
6100 // Incoming resolution increases, initial frame drop activates.
6101 // Frame should be dropped, link allocation too low for frame.
6102 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6103 ExpectDroppedFrame();
6104
6105 // Expect sink_wants to specify a scaled frame.
6106 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6107 5000);
6108 video_stream_encoder_->Stop();
6109}
6110
6111TEST_F(VideoStreamEncoderTest,
6112 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6113 const int kMinStartBps360p = 222000;
6114 fake_encoder_.SetResolutionBitrateLimits(
6115 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6116 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6117 800000)});
6118
6119 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6120 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6121 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6122 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6123 0, 0, 0);
6124 // Frame should not be dropped, bitrate not too low for frame.
6125 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6126 WaitForEncodedFrame(1);
6127
6128 // Incoming resolution increases, initial frame drop activates.
6129 // Frame should be dropped, link allocation not too low for frame.
6130 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6131 WaitForEncodedFrame(2);
6132
6133 video_stream_encoder_->Stop();
6134}
6135
Åsa Perssone644a032019-11-08 15:56:00 +01006136TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006137 webrtc::test::ScopedKeyValueConfig field_trials(
6138 field_trials_,
Åsa Persson06defc42021-09-10 15:28:48 +02006139 "WebRTC-Video-QualityRampupSettings/"
6140 "min_pixels:921600,min_duration_ms:2000/");
6141
6142 const int kWidth = 1280;
6143 const int kHeight = 720;
6144 const int kFps = 10;
6145 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006146
6147 // Reset encoder for field trials to take effect.
6148 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 10:07:11 +02006149 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 11:34:32 +02006150 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01006151 ConfigureEncoder(std::move(config));
6152 fake_encoder_.SetQp(kQpLow);
6153
6154 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006155 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01006156 source.set_adaptation_enabled(true);
6157 video_stream_encoder_->SetSource(&source,
6158 DegradationPreference::MAINTAIN_FRAMERATE);
6159
6160 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 10:07:11 +02006161 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 18:49:07 +02006162 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006163 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006164
6165 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 15:28:48 +02006166 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006167 int64_t timestamp_ms = kFrameIntervalMs;
6168 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6169 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02006170 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6171 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01006172
6173 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02006174 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6175 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006176
Artem Titovab30d722021-07-27 16:22:11 +02006177 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 15:28:48 +02006178 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 15:56:00 +01006179 for (size_t i = 1; i <= 10; i++) {
6180 timestamp_ms += kFrameIntervalMs;
6181 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6182 WaitForEncodedFrame(timestamp_ms);
6183 }
Åsa Persson06defc42021-09-10 15:28:48 +02006184
6185 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6186 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6187 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6188 timestamp_ms += kFrameIntervalMs;
6189 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6190 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 15:56:00 +01006191 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6192 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6193
Åsa Persson06defc42021-09-10 15:28:48 +02006194 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 15:56:00 +01006195 timestamp_ms += kFrameIntervalMs;
6196 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6197 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006198 // The ramp-up code involves the adaptation queue, give it time to execute.
6199 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006200 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006201 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006202
6203 // Frame should not be adapted.
6204 timestamp_ms += kFrameIntervalMs;
6205 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6206 WaitForEncodedFrame(kWidth, kHeight);
6207 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6208
6209 video_stream_encoder_->Stop();
6210}
6211
mflodmancc3d4422017-08-03 08:27:51 -07006212TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006213 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006214 webrtc::test::ScopedKeyValueConfig field_trials(
6215 field_trials_, "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006216 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006217 source.set_adaptation_enabled(true);
6218 video_stream_encoder_->SetSource(&source,
6219 DegradationPreference::MAINTAIN_FRAMERATE);
6220 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006221 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006222 fake_encoder_.SetQp(kQpHigh + 1);
6223 const int kWidth = 1280;
6224 const int kHeight = 720;
6225 const int64_t kFrameIntervalMs = 100;
6226 int64_t timestamp_ms = kFrameIntervalMs;
6227 for (size_t i = 1; i <= 100; i++) {
6228 timestamp_ms += kFrameIntervalMs;
6229 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6230 WaitForEncodedFrame(timestamp_ms);
6231 }
6232 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6233 // for the first time.
6234 // TODO(eshr): We should avoid these waits by using threads with simulated
6235 // time.
6236 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6237 2000 * 2.5 * 2);
6238 timestamp_ms += kFrameIntervalMs;
6239 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6240 WaitForEncodedFrame(timestamp_ms);
6241 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6242 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6243 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6244
6245 // Disable Quality scaling by turning off scaler on the encoder and
6246 // reconfiguring.
6247 fake_encoder_.SetQualityScaling(false);
6248 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6249 kMaxPayloadLength);
6250 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Markus Handell28c71802021-11-08 10:11:55 +01006251 AdvanceTime(TimeDelta::Zero());
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006252 // Since we turned off the quality scaler, the adaptations made by it are
6253 // removed.
6254 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6255 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6256
6257 video_stream_encoder_->Stop();
6258}
6259
6260TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006261 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6262 const int kTooSmallWidth = 10;
6263 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006264 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006265 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006266
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006267 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006268 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006269 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006270 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006271 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006272 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6273
6274 // Trigger adapt down, too small frame, expect no change.
6275 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006276 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006277 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006278 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006279 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6280 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6281
mflodmancc3d4422017-08-03 08:27:51 -07006282 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006283}
6284
mflodmancc3d4422017-08-03 08:27:51 -07006285TEST_F(VideoStreamEncoderTest,
6286 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006287 const int kTooSmallWidth = 10;
6288 const int kTooSmallHeight = 10;
6289 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006290 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006291 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006292
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006293 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006294 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006295 video_stream_encoder_->SetSource(&source,
6296 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006297 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006298 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6299 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6300
6301 // Trigger adapt down, expect limited framerate.
6302 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006303 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006304 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006305 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006306 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6307 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6308 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6309
6310 // Trigger adapt down, too small frame, expect no change.
6311 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006312 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006313 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006314 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006315 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6316 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6317 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6318
mflodmancc3d4422017-08-03 08:27:51 -07006319 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006320}
6321
mflodmancc3d4422017-08-03 08:27:51 -07006322TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006323 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006324 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006325 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006326 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006327 const int kFrameWidth = 1280;
6328 const int kFrameHeight = 720;
6329 video_source_.IncomingCapturedFrame(
6330 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006331 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006332 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006333}
6334
sprangb1ca0732017-02-01 08:38:12 -08006335// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006336TEST_F(VideoStreamEncoderTest,
6337 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006338 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006339 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006340
6341 const int kFrameWidth = 1280;
6342 const int kFrameHeight = 720;
6343 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006344 // requested by
6345 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006346 video_source_.set_adaptation_enabled(true);
6347
6348 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006349 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006350 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006351
6352 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006353 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006354 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006355 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006356 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006357
asaperssonfab67072017-04-04 05:51:49 -07006358 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006359 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006360 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006361 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006362 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006363
mflodmancc3d4422017-08-03 08:27:51 -07006364 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006365}
sprangfe627f32017-03-29 08:24:59 -07006366
mflodmancc3d4422017-08-03 08:27:51 -07006367TEST_F(VideoStreamEncoderTest,
6368 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006369 const int kFrameWidth = 1280;
6370 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006371
Henrik Boström381d1092020-05-12 18:49:07 +02006372 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006373 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006374 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006375 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006376 video_source_.set_adaptation_enabled(true);
6377
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006378 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006379
6380 video_source_.IncomingCapturedFrame(
6381 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006382 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006383
6384 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006385 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006386
6387 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006388 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006389 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006390 video_source_.IncomingCapturedFrame(
6391 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006392 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006393 }
6394
6395 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006396 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006397 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006398 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006399 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006400 video_source_.IncomingCapturedFrame(
6401 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Markus Handell2cfc1af2022-08-19 08:16:48 +00006402 if (!WaitForFrame(kFrameTimeout)) {
sprangc5d62e22017-04-02 23:53:04 -07006403 ++num_frames_dropped;
6404 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006405 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006406 }
6407 }
6408
sprang4847ae62017-06-27 07:06:52 -07006409 // Add some slack to account for frames dropped by the frame dropper.
6410 const int kErrorMargin = 1;
6411 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006412 kErrorMargin);
6413
6414 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006415 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006416 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006417 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006418 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006419 video_source_.IncomingCapturedFrame(
6420 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Markus Handell2cfc1af2022-08-19 08:16:48 +00006421 if (!WaitForFrame(kFrameTimeout)) {
sprangc5d62e22017-04-02 23:53:04 -07006422 ++num_frames_dropped;
6423 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006424 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006425 }
6426 }
sprang4847ae62017-06-27 07:06:52 -07006427 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006428 kErrorMargin);
6429
6430 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006431 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006432 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006433 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006434 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006435 video_source_.IncomingCapturedFrame(
6436 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Markus Handell2cfc1af2022-08-19 08:16:48 +00006437 if (!WaitForFrame(kFrameTimeout)) {
sprangc5d62e22017-04-02 23:53:04 -07006438 ++num_frames_dropped;
6439 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006440 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006441 }
6442 }
sprang4847ae62017-06-27 07:06:52 -07006443 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006444 kErrorMargin);
6445
6446 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006447 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006448 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006449 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006450 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006451 video_source_.IncomingCapturedFrame(
6452 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Markus Handell2cfc1af2022-08-19 08:16:48 +00006453 if (!WaitForFrame(kFrameTimeout)) {
sprangc5d62e22017-04-02 23:53:04 -07006454 ++num_frames_dropped;
6455 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006456 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006457 }
6458 }
6459 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6460
mflodmancc3d4422017-08-03 08:27:51 -07006461 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006462}
6463
mflodmancc3d4422017-08-03 08:27:51 -07006464TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006465 const int kFramerateFps = 5;
6466 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006467 const int kFrameWidth = 1280;
6468 const int kFrameHeight = 720;
6469
sprang4847ae62017-06-27 07:06:52 -07006470 // Reconfigure encoder with two temporal layers and screensharing, which will
6471 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006472 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006473
Henrik Boström381d1092020-05-12 18:49:07 +02006474 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006475 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006476 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006477 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006478 video_source_.set_adaptation_enabled(true);
6479
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006480 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006481
6482 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006483 rtc::VideoSinkWants last_wants;
6484 do {
6485 last_wants = video_source_.sink_wants();
6486
sprangc5d62e22017-04-02 23:53:04 -07006487 // Insert frames to get a new fps estimate...
6488 for (int j = 0; j < kFramerateFps; ++j) {
6489 video_source_.IncomingCapturedFrame(
6490 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006491 if (video_source_.last_sent_width()) {
6492 sink_.WaitForEncodedFrame(timestamp_ms);
6493 }
sprangc5d62e22017-04-02 23:53:04 -07006494 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006495 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006496 }
6497 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006498 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006499 } while (video_source_.sink_wants().max_framerate_fps <
6500 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006501
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006502 EXPECT_THAT(video_source_.sink_wants(),
6503 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006504
mflodmancc3d4422017-08-03 08:27:51 -07006505 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006506}
asaperssonf7e294d2017-06-13 23:25:22 -07006507
mflodmancc3d4422017-08-03 08:27:51 -07006508TEST_F(VideoStreamEncoderTest,
6509 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006510 const int kWidth = 1280;
6511 const int kHeight = 720;
6512 const int64_t kFrameIntervalMs = 150;
6513 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006514 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006515 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006516
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006517 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006518 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006519 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006520 video_stream_encoder_->SetSource(&source,
6521 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006522 timestamp_ms += kFrameIntervalMs;
6523 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006524 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006525 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006526 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6527 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6528 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6529
6530 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006531 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006532 timestamp_ms += kFrameIntervalMs;
6533 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006534 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006535 EXPECT_THAT(source.sink_wants(),
6536 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006537 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6538 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6539 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6540
6541 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006542 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006543 timestamp_ms += kFrameIntervalMs;
6544 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006545 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006546 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006547 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6548 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6549 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6550
6551 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006552 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006553 timestamp_ms += kFrameIntervalMs;
6554 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006555 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006556 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006557 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6558 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6559 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6560
6561 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006562 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006563 timestamp_ms += kFrameIntervalMs;
6564 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006565 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006566 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006567 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6568 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6569 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6570
6571 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006572 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006573 timestamp_ms += kFrameIntervalMs;
6574 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006575 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006576 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006577 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6578 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6579 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6580
6581 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006582 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006583 timestamp_ms += kFrameIntervalMs;
6584 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006585 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006586 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006587 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6588 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6589 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6590
6591 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006592 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006593 timestamp_ms += kFrameIntervalMs;
6594 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006595 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006596 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006597 rtc::VideoSinkWants last_wants = source.sink_wants();
6598 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6599 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6600 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6601
6602 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006603 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006604 timestamp_ms += kFrameIntervalMs;
6605 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006606 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006607 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006608 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6609 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6610 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6611
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006612 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006613 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006614 timestamp_ms += kFrameIntervalMs;
6615 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006616 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006617 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006618 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6619 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6620 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6621
6622 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006623 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006624 timestamp_ms += kFrameIntervalMs;
6625 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006626 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006627 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006628 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6629 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6630 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6631
6632 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006633 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006634 timestamp_ms += kFrameIntervalMs;
6635 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006636 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006637 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006638 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6639 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6640 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6641
6642 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006643 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006644 timestamp_ms += kFrameIntervalMs;
6645 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006646 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006647 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006648 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6649 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6650 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6651
6652 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006653 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006654 timestamp_ms += kFrameIntervalMs;
6655 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006656 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006657 EXPECT_THAT(source.sink_wants(), FpsMax());
6658 EXPECT_EQ(source.sink_wants().max_pixel_count,
6659 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006660 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6661 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6662 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6663
6664 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006665 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006666 timestamp_ms += kFrameIntervalMs;
6667 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006668 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006669 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006670 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6671 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6672 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6673
Åsa Persson30ab0152019-08-27 12:22:33 +02006674 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006675 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006676 timestamp_ms += kFrameIntervalMs;
6677 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006678 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006679 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006680 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006681 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6682 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6683 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6684
6685 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006686 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006687 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006688 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6689
mflodmancc3d4422017-08-03 08:27:51 -07006690 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006691}
6692
mflodmancc3d4422017-08-03 08:27:51 -07006693TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006694 const int kWidth = 1280;
6695 const int kHeight = 720;
6696 const int64_t kFrameIntervalMs = 150;
6697 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006698 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006699 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006700
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006701 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006702 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006703 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006704 video_stream_encoder_->SetSource(&source,
6705 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006706 timestamp_ms += kFrameIntervalMs;
6707 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006708 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006709 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006710 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6711 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6712 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6713 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6714 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6715 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6716
6717 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006718 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006719 timestamp_ms += kFrameIntervalMs;
6720 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006721 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006722 EXPECT_THAT(source.sink_wants(),
6723 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006724 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6725 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6726 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6727 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6728 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6729 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6730
6731 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006732 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006733 timestamp_ms += kFrameIntervalMs;
6734 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006735 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006736 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006737 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6738 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6739 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6740 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6741 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6742 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6743
6744 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006745 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006746 timestamp_ms += kFrameIntervalMs;
6747 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006748 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006749 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006750 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006751 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6752 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6753 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6754 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6755 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6756
Evan Shrubsole64469032020-06-11 10:45:29 +02006757 // Trigger cpu adapt up, expect no change since QP is most limited.
6758 {
6759 // Store current sink wants since we expect no change and if there is no
6760 // change then last_wants() is not updated.
6761 auto previous_sink_wants = source.sink_wants();
6762 video_stream_encoder_->TriggerCpuUnderuse();
6763 timestamp_ms += kFrameIntervalMs;
6764 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6765 WaitForEncodedFrame(timestamp_ms);
6766 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6767 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6768 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6769 }
6770
6771 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6772 video_stream_encoder_->TriggerQualityHigh();
6773 timestamp_ms += kFrameIntervalMs;
6774 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6775 WaitForEncodedFrame(timestamp_ms);
6776 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6777 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6778 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6779 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6780 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6781 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6782 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6783
6784 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6785 // expect increased resolution (960x540@30fps).
6786 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006787 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006788 timestamp_ms += kFrameIntervalMs;
6789 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006790 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006791 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006792 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6793 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6794 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6795 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6796 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006797 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006798
Evan Shrubsole64469032020-06-11 10:45:29 +02006799 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6800 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006801 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006802 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006803 timestamp_ms += kFrameIntervalMs;
6804 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006805 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006806 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006807 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006808 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6809 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6810 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6811 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6812 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006813 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006814
6815 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006816 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006817 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006818 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006819 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006820
mflodmancc3d4422017-08-03 08:27:51 -07006821 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006822}
6823
mflodmancc3d4422017-08-03 08:27:51 -07006824TEST_F(VideoStreamEncoderTest,
6825 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006826 const int kWidth = 640;
6827 const int kHeight = 360;
6828 const int kFpsLimit = 15;
6829 const int64_t kFrameIntervalMs = 150;
6830 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006831 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006832 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006833
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006834 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006835 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006836 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006837 video_stream_encoder_->SetSource(&source,
6838 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006839 timestamp_ms += kFrameIntervalMs;
6840 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006841 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006842 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006843 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6844 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6845 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6846 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6847 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6848 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6849
6850 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006851 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006852 timestamp_ms += kFrameIntervalMs;
6853 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006854 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006855 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006856 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6857 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6858 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6859 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6860 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6861 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6862
6863 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006864 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006865 timestamp_ms += kFrameIntervalMs;
6866 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006867 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006868 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006869 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006870 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006871 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6872 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6873 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6874 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6875
Evan Shrubsole64469032020-06-11 10:45:29 +02006876 // Trigger cpu adapt up, expect no change because quality is most limited.
6877 {
6878 auto previous_sink_wants = source.sink_wants();
6879 // Store current sink wants since we expect no change ind if there is no
6880 // change then last__wants() is not updated.
6881 video_stream_encoder_->TriggerCpuUnderuse();
6882 timestamp_ms += kFrameIntervalMs;
6883 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6884 WaitForEncodedFrame(timestamp_ms);
6885 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6886 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6887 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6888 }
6889
6890 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6891 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006892 timestamp_ms += kFrameIntervalMs;
6893 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006894 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006895 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006896 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6897 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6898 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006899 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6900 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6901 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006902
Evan Shrubsole64469032020-06-11 10:45:29 +02006903 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006904 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006905 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006906 timestamp_ms += kFrameIntervalMs;
6907 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006908 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006909 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006910 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6911 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6912 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6913 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6914 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006915 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006916
6917 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006918 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006919 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006920 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006921 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006922
mflodmancc3d4422017-08-03 08:27:51 -07006923 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006924}
6925
mflodmancc3d4422017-08-03 08:27:51 -07006926TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006927 const int kFrameWidth = 1920;
6928 const int kFrameHeight = 1080;
Ying Wangdb0d5862022-04-28 19:26:22 +02006929 // 2/3 of 1920.
6930 const int kAdaptedFrameWidth = 1280;
6931 // 2/3 of 1080.
6932 const int kAdaptedFrameHeight = 720;
ilnik6b826ef2017-06-16 06:53:48 -07006933 const int kFramerate = 24;
6934
Henrik Boström381d1092020-05-12 18:49:07 +02006935 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006936 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006937 // Trigger reconfigure encoder (without resetting the entire instance).
6938 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006939 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6940 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02006941 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 06:53:48 -07006942 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006943 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006944 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006945 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006946 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006947
6948 video_source_.set_adaptation_enabled(true);
6949
6950 video_source_.IncomingCapturedFrame(
6951 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006952 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006953
6954 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006955 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006956 video_source_.IncomingCapturedFrame(
6957 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006958 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006959
mflodmancc3d4422017-08-03 08:27:51 -07006960 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006961}
6962
mflodmancc3d4422017-08-03 08:27:51 -07006963TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006964 const int kFrameWidth = 1280;
6965 const int kFrameHeight = 720;
6966 const int kLowFps = 2;
6967 const int kHighFps = 30;
6968
Henrik Boström381d1092020-05-12 18:49:07 +02006969 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006970 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006971
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006972 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006973 max_framerate_ = kLowFps;
6974
6975 // Insert 2 seconds of 2fps video.
6976 for (int i = 0; i < kLowFps * 2; ++i) {
6977 video_source_.IncomingCapturedFrame(
6978 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6979 WaitForEncodedFrame(timestamp_ms);
6980 timestamp_ms += 1000 / kLowFps;
6981 }
6982
6983 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006984 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006985 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006986 video_source_.IncomingCapturedFrame(
6987 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6988 WaitForEncodedFrame(timestamp_ms);
6989 timestamp_ms += 1000 / kLowFps;
6990
6991 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6992
6993 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006994 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
Markus Handell2cfc1af2022-08-19 08:16:48 +00006995 constexpr TimeDelta kFrameInterval = TimeDelta::Seconds(1) / kHighFps;
sprang4847ae62017-06-27 07:06:52 -07006996 max_framerate_ = kHighFps;
6997 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6998 video_source_.IncomingCapturedFrame(
6999 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7000 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
7001 // be dropped if the encoder hans't been updated with the new higher target
7002 // framerate yet, causing it to overshoot the target bitrate and then
7003 // suffering the wrath of the media optimizer.
Markus Handell2cfc1af2022-08-19 08:16:48 +00007004 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameInterval);
7005 timestamp_ms += kFrameInterval.ms();
sprang4847ae62017-06-27 07:06:52 -07007006 }
7007
7008 // Don expect correct measurement just yet, but it should be higher than
7009 // before.
7010 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
7011
mflodmancc3d4422017-08-03 08:27:51 -07007012 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007013}
7014
mflodmancc3d4422017-08-03 08:27:51 -07007015TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07007016 const int kFrameWidth = 1280;
7017 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 15:09:05 +02007018 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01007019 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02007020 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07007021
Henrik Boström381d1092020-05-12 18:49:07 +02007022 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007023 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07007024 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07007025
7026 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007027 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07007028 video_source_.IncomingCapturedFrame(
7029 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7030 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02007031 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007032
7033 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02007034 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007035 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07007036
7037 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02007038 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007039 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07007040
Per Kjellanderdcef6412020-10-07 15:09:05 +02007041 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07007042 video_source_.IncomingCapturedFrame(
7043 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7044 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02007045 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007046
mflodmancc3d4422017-08-03 08:27:51 -07007047 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007048}
ilnik6b826ef2017-06-16 06:53:48 -07007049
Niels Möller4db138e2018-04-19 09:04:13 +02007050TEST_F(VideoStreamEncoderTest,
7051 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
7052 const int kFrameWidth = 1280;
7053 const int kFrameHeight = 720;
Markus Handell8e4197b2022-05-30 15:45:28 +02007054 const test::ScopedKeyValueConfig kFieldTrials;
7055 const CpuOveruseOptions default_options(kFieldTrials);
Henrik Boström381d1092020-05-12 18:49:07 +02007056 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007057 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007058 video_source_.IncomingCapturedFrame(
7059 CreateFrame(1, kFrameWidth, kFrameHeight));
7060 WaitForEncodedFrame(1);
7061 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7062 .low_encode_usage_threshold_percent,
7063 default_options.low_encode_usage_threshold_percent);
7064 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7065 .high_encode_usage_threshold_percent,
7066 default_options.high_encode_usage_threshold_percent);
7067 video_stream_encoder_->Stop();
7068}
7069
7070TEST_F(VideoStreamEncoderTest,
7071 HigherCpuAdaptationThresholdsForHardwareEncoder) {
7072 const int kFrameWidth = 1280;
7073 const int kFrameHeight = 720;
Markus Handell8e4197b2022-05-30 15:45:28 +02007074 const test::ScopedKeyValueConfig kFieldTrials;
7075 CpuOveruseOptions hardware_options(kFieldTrials);
Niels Möller4db138e2018-04-19 09:04:13 +02007076 hardware_options.low_encode_usage_threshold_percent = 150;
7077 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01007078 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02007079
Henrik Boström381d1092020-05-12 18:49:07 +02007080 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007081 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007082 video_source_.IncomingCapturedFrame(
7083 CreateFrame(1, kFrameWidth, kFrameHeight));
7084 WaitForEncodedFrame(1);
7085 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7086 .low_encode_usage_threshold_percent,
7087 hardware_options.low_encode_usage_threshold_percent);
7088 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7089 .high_encode_usage_threshold_percent,
7090 hardware_options.high_encode_usage_threshold_percent);
7091 video_stream_encoder_->Stop();
7092}
7093
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007094TEST_F(VideoStreamEncoderTest,
7095 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
7096 const int kFrameWidth = 1280;
7097 const int kFrameHeight = 720;
7098
Markus Handell8e4197b2022-05-30 15:45:28 +02007099 const test::ScopedKeyValueConfig kFieldTrials;
7100 const CpuOveruseOptions default_options(kFieldTrials);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007101 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007102 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007103 video_source_.IncomingCapturedFrame(
7104 CreateFrame(1, kFrameWidth, kFrameHeight));
7105 WaitForEncodedFrame(1);
7106 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7107 .low_encode_usage_threshold_percent,
7108 default_options.low_encode_usage_threshold_percent);
7109 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7110 .high_encode_usage_threshold_percent,
7111 default_options.high_encode_usage_threshold_percent);
7112
Markus Handell8e4197b2022-05-30 15:45:28 +02007113 CpuOveruseOptions hardware_options(kFieldTrials);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007114 hardware_options.low_encode_usage_threshold_percent = 150;
7115 hardware_options.high_encode_usage_threshold_percent = 200;
7116 fake_encoder_.SetIsHardwareAccelerated(true);
7117
7118 video_source_.IncomingCapturedFrame(
7119 CreateFrame(2, kFrameWidth, kFrameHeight));
7120 WaitForEncodedFrame(2);
7121
7122 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7123 .low_encode_usage_threshold_percent,
7124 hardware_options.low_encode_usage_threshold_percent);
7125 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7126 .high_encode_usage_threshold_percent,
7127 hardware_options.high_encode_usage_threshold_percent);
7128
7129 video_stream_encoder_->Stop();
7130}
7131
Niels Möller6bb5ab92019-01-11 11:11:10 +01007132TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7133 const int kFrameWidth = 320;
7134 const int kFrameHeight = 240;
7135 const int kFps = 30;
Asa Persson606d3cb2021-10-04 10:07:11 +02007136 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007137 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7138
Henrik Boström381d1092020-05-12 18:49:07 +02007139 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007140 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007141
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007142 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007143 max_framerate_ = kFps;
7144
7145 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7146 fake_encoder_.SimulateOvershoot(1.0);
7147 int num_dropped = 0;
7148 for (int i = 0; i < kNumFramesInRun; ++i) {
7149 video_source_.IncomingCapturedFrame(
7150 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7151 // Wait up to two frame durations for a frame to arrive.
Markus Handell2cfc1af2022-08-19 08:16:48 +00007152 if (!TimedWaitForEncodedFrame(timestamp_ms,
7153 2 * TimeDelta::Seconds(1) / kFps)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01007154 ++num_dropped;
7155 }
7156 timestamp_ms += 1000 / kFps;
7157 }
7158
Erik Språnga8d48ab2019-02-08 14:17:40 +01007159 // Framerate should be measured to be near the expected target rate.
7160 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7161
7162 // Frame drops should be within 5% of expected 0%.
7163 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007164
7165 // Make encoder produce frames at double the expected bitrate during 3 seconds
7166 // of video, verify number of drops. Rate needs to be slightly changed in
7167 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01007168 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02007169 const RateControlSettings trials =
7170 RateControlSettings::ParseFromFieldTrials();
7171 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01007172 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02007173 // frame dropping since the adjuter will try to just lower the target
7174 // bitrate rather than drop frames. If network headroom can be used, it
7175 // doesn't push back as hard so we don't need quite as much overshoot.
7176 // These numbers are unfortunately a bit magical but there's not trivial
7177 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01007178 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01007179 }
7180 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02007181 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007182 kTargetBitrate + DataRate::KilobitsPerSec(1),
7183 kTargetBitrate + DataRate::KilobitsPerSec(1),
7184 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007185 num_dropped = 0;
7186 for (int i = 0; i < kNumFramesInRun; ++i) {
7187 video_source_.IncomingCapturedFrame(
7188 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7189 // Wait up to two frame durations for a frame to arrive.
Markus Handell2cfc1af2022-08-19 08:16:48 +00007190 if (!TimedWaitForEncodedFrame(timestamp_ms,
7191 2 * TimeDelta::Seconds(1) / kFps)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01007192 ++num_dropped;
7193 }
7194 timestamp_ms += 1000 / kFps;
7195 }
7196
Henrik Boström381d1092020-05-12 18:49:07 +02007197 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007198 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007199
7200 // Target framerate should be still be near the expected target, despite
7201 // the frame drops.
7202 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7203
7204 // Frame drops should be within 5% of expected 50%.
7205 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007206
7207 video_stream_encoder_->Stop();
7208}
7209
7210TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7211 const int kFrameWidth = 320;
7212 const int kFrameHeight = 240;
7213 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 10:07:11 +02007214 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007215
7216 ASSERT_GT(max_framerate_, kActualInputFps);
7217
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007218 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007219 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007220 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007221 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007222
7223 // Insert 3 seconds of video, with an input fps lower than configured max.
7224 for (int i = 0; i < kActualInputFps * 3; ++i) {
7225 video_source_.IncomingCapturedFrame(
7226 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7227 // Wait up to two frame durations for a frame to arrive.
7228 WaitForEncodedFrame(timestamp_ms);
7229 timestamp_ms += 1000 / kActualInputFps;
7230 }
7231
7232 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7233
7234 video_stream_encoder_->Stop();
7235}
7236
Markus Handell9a478b52021-11-18 16:07:01 +01007237TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007238 VideoFrame::UpdateRect rect;
Markus Handell9a478b52021-11-18 16:07:01 +01007239 test::FrameForwarder source;
7240 video_stream_encoder_->SetSource(&source,
7241 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02007242 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007243 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007244
Markus Handell9a478b52021-11-18 16:07:01 +01007245 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(1, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007246 WaitForEncodedFrame(1);
7247 // On the very first frame full update should be forced.
7248 rect = fake_encoder_.GetLastUpdateRect();
7249 EXPECT_EQ(rect.offset_x, 0);
7250 EXPECT_EQ(rect.offset_y, 0);
7251 EXPECT_EQ(rect.height, codec_height_);
7252 EXPECT_EQ(rect.width, codec_width_);
Markus Handell9a478b52021-11-18 16:07:01 +01007253 // Frame with NTP timestamp 2 will be dropped due to outstanding frames
7254 // scheduled for processing during encoder queue processing of frame 2.
7255 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(2, nullptr, 1));
7256 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(3, nullptr, 10));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007257 WaitForEncodedFrame(3);
7258 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7259 rect = fake_encoder_.GetLastUpdateRect();
7260 EXPECT_EQ(rect.offset_x, 1);
7261 EXPECT_EQ(rect.offset_y, 0);
7262 EXPECT_EQ(rect.width, 10);
7263 EXPECT_EQ(rect.height, 1);
7264
Markus Handell9a478b52021-11-18 16:07:01 +01007265 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(4, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007266 WaitForEncodedFrame(4);
7267 // Previous frame was encoded, so no accumulation should happen.
7268 rect = fake_encoder_.GetLastUpdateRect();
7269 EXPECT_EQ(rect.offset_x, 0);
7270 EXPECT_EQ(rect.offset_y, 0);
7271 EXPECT_EQ(rect.width, 1);
7272 EXPECT_EQ(rect.height, 1);
7273
7274 video_stream_encoder_->Stop();
7275}
7276
Erik Språngd7329ca2019-02-21 21:19:53 +01007277TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007278 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007279 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007280
7281 // First frame is always keyframe.
7282 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7283 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007284 EXPECT_THAT(
7285 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007286 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007287
7288 // Insert delta frame.
7289 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7290 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007291 EXPECT_THAT(
7292 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007293 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007294
7295 // Request next frame be a key-frame.
7296 video_stream_encoder_->SendKeyFrame();
7297 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7298 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007299 EXPECT_THAT(
7300 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007301 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007302
7303 video_stream_encoder_->Stop();
7304}
7305
7306TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7307 // Setup simulcast with three streams.
7308 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007309 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007310 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7311 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007312 // Wait for all three layers before triggering event.
7313 sink_.SetNumExpectedLayers(3);
7314
7315 // First frame is always keyframe.
7316 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7317 WaitForEncodedFrame(1);
7318 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007319 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7320 VideoFrameType::kVideoFrameKey,
7321 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007322
7323 // Insert delta frame.
7324 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7325 WaitForEncodedFrame(2);
7326 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007327 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7328 VideoFrameType::kVideoFrameDelta,
7329 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007330
7331 // Request next frame be a key-frame.
7332 // Only first stream is configured to produce key-frame.
7333 video_stream_encoder_->SendKeyFrame();
7334 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7335 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007336
7337 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7338 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007339 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007340 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007341 VideoFrameType::kVideoFrameKey,
7342 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007343
7344 video_stream_encoder_->Stop();
7345}
7346
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007347TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007348 // SPS contains VUI with restrictions on the maximum number of reordered
7349 // pictures, there is no need to rewrite the bitstream to enable faster
7350 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007351 ResetEncoder("H264", 1, 1, 1, false);
7352
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007353 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007354 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007355 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007356
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007357 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 10:07:11 +02007358 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007359
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007360 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7361 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007362
7363 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007364 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007365
7366 video_stream_encoder_->Stop();
7367}
7368
7369TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007370 // SPS does not contain VUI, the bitstream is will be rewritten with added
7371 // VUI with restrictions on the maximum number of reordered pictures to
7372 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007373 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7374 0x00, 0x00, 0x03, 0x03, 0xF4,
7375 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007376 ResetEncoder("H264", 1, 1, 1, false);
7377
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007378 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007379 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007380 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007381
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007382 fake_encoder_.SetEncodedImageData(
7383 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007384
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007385 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7386 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007387
7388 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007389 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007390
7391 video_stream_encoder_->Stop();
7392}
7393
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007394TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7395 const int kFrameWidth = 1280;
7396 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 10:07:11 +02007397 const DataRate kTargetBitrate =
7398 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007399
Henrik Boström381d1092020-05-12 18:49:07 +02007400 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007401 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007402 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7403
7404 // Insert a first video frame. It should be dropped because of downscale in
7405 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007406 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007407 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7408 frame.set_rotation(kVideoRotation_270);
7409 video_source_.IncomingCapturedFrame(frame);
7410
7411 ExpectDroppedFrame();
7412
7413 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007414 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007415 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7416 frame.set_rotation(kVideoRotation_90);
7417 video_source_.IncomingCapturedFrame(frame);
7418
7419 WaitForEncodedFrame(timestamp_ms);
7420 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7421
7422 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007423 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007424 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7425 frame.set_rotation(kVideoRotation_180);
7426 video_source_.IncomingCapturedFrame(frame);
7427
7428 WaitForEncodedFrame(timestamp_ms);
7429 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7430
7431 video_stream_encoder_->Stop();
7432}
7433
Erik Språng5056af02019-09-02 15:53:11 +02007434TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7435 const int kFrameWidth = 320;
7436 const int kFrameHeight = 180;
7437
7438 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007439 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007440 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7441 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7442 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007443 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007444 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007445 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007446
7447 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007448 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007449 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7450 frame.set_rotation(kVideoRotation_270);
7451 video_source_.IncomingCapturedFrame(frame);
7452 WaitForEncodedFrame(timestamp_ms);
7453
7454 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 10:07:11 +02007455 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007456 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7457 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007458 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007459 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007460 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007461 /*link_allocation=*/target_rate,
7462 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007463 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007464 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007465 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7466
7467 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7468 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7469 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007470 DataRate allocation_sum =
7471 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007472 EXPECT_EQ(min_rate, allocation_sum);
7473 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7474
7475 video_stream_encoder_->Stop();
7476}
7477
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007478TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007479 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007480 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007481 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007482 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007483 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7484 WaitForEncodedFrame(1);
7485
7486 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7487 ASSERT_TRUE(prev_rate_settings.has_value());
7488 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7489 kDefaultFramerate);
7490
7491 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7492 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7493 timestamp_ms += 1000 / kDefaultFramerate;
7494 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7495 WaitForEncodedFrame(timestamp_ms);
7496 }
7497 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7498 kDefaultFramerate);
7499 // Capture larger frame to trigger a reconfigure.
7500 codec_height_ *= 2;
7501 codec_width_ *= 2;
7502 timestamp_ms += 1000 / kDefaultFramerate;
7503 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7504 WaitForEncodedFrame(timestamp_ms);
7505
7506 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7507 auto current_rate_settings =
7508 fake_encoder_.GetAndResetLastRateControlSettings();
7509 // Ensure we have actually reconfigured twice
7510 // The rate settings should have been set again even though
7511 // they haven't changed.
7512 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007513 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007514
7515 video_stream_encoder_->Stop();
7516}
7517
philipeld9cc8c02019-09-16 14:53:40 +02007518struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007519 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007520 MOCK_METHOD(void,
7521 RequestEncoderSwitch,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007522 (const webrtc::SdpVideoFormat& format,
7523 bool allow_default_fallback),
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007524 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007525};
7526
philipel9b058032020-02-10 11:30:00 +01007527TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7528 constexpr int kDontCare = 100;
7529 StrictMock<MockEncoderSelector> encoder_selector;
7530 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7531 &fake_encoder_, &encoder_selector);
7532 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7533
7534 // Reset encoder for new configuration to take effect.
7535 ConfigureEncoder(video_encoder_config_.Copy());
7536
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007537 EXPECT_CALL(encoder_selector, OnCurrentEncoder);
philipel9b058032020-02-10 11:30:00 +01007538
7539 video_source_.IncomingCapturedFrame(
7540 CreateFrame(kDontCare, kDontCare, kDontCare));
Markus Handell28c71802021-11-08 10:11:55 +01007541 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007542 video_stream_encoder_->Stop();
7543
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007544 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007545 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007546 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7547 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007548 video_stream_encoder_.reset();
7549}
7550
7551TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7552 constexpr int kDontCare = 100;
7553
7554 NiceMock<MockEncoderSelector> encoder_selector;
7555 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7556 video_send_config_.encoder_settings.encoder_switch_request_callback =
7557 &switch_callback;
7558 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7559 &fake_encoder_, &encoder_selector);
7560 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7561
7562 // Reset encoder for new configuration to take effect.
7563 ConfigureEncoder(video_encoder_config_.Copy());
7564
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007565 ON_CALL(encoder_selector, OnAvailableBitrate)
philipel9b058032020-02-10 11:30:00 +01007566 .WillByDefault(Return(SdpVideoFormat("AV1")));
7567 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007568 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7569 /*allow_default_fallback=*/false));
philipel9b058032020-02-10 11:30:00 +01007570
Henrik Boström381d1092020-05-12 18:49:07 +02007571 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007572 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7573 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7574 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007575 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007576 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007577 /*cwnd_reduce_ratio=*/0);
Markus Handell28c71802021-11-08 10:11:55 +01007578 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007579
7580 video_stream_encoder_->Stop();
7581}
7582
philipel6daa3042022-04-11 10:48:28 +02007583TEST_F(VideoStreamEncoderTest, EncoderSelectorResolutionSwitch) {
7584 NiceMock<MockEncoderSelector> encoder_selector;
7585 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7586 video_send_config_.encoder_settings.encoder_switch_request_callback =
7587 &switch_callback;
7588 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7589 &fake_encoder_, &encoder_selector);
7590 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7591
7592 // Reset encoder for new configuration to take effect.
7593 ConfigureEncoder(video_encoder_config_.Copy());
7594
7595 EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(640, 480)))
7596 .WillOnce(Return(absl::nullopt));
7597 EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(320, 240)))
7598 .WillOnce(Return(SdpVideoFormat("AV1")));
7599 EXPECT_CALL(switch_callback,
7600 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7601 /*allow_default_fallback=*/false));
7602
7603 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7604 /*target_bitrate=*/DataRate::KilobitsPerSec(800),
7605 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(1000),
7606 /*link_allocation=*/DataRate::KilobitsPerSec(1000),
7607 /*fraction_lost=*/0,
7608 /*round_trip_time_ms=*/0,
7609 /*cwnd_reduce_ratio=*/0);
7610
7611 video_source_.IncomingCapturedFrame(CreateFrame(1, 640, 480));
7612 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 480));
7613 video_source_.IncomingCapturedFrame(CreateFrame(3, 320, 240));
7614
7615 AdvanceTime(TimeDelta::Zero());
7616
7617 video_stream_encoder_->Stop();
7618}
7619
philipel9b058032020-02-10 11:30:00 +01007620TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7621 constexpr int kSufficientBitrateToNotDrop = 1000;
7622 constexpr int kDontCare = 100;
7623
7624 NiceMock<MockVideoEncoder> video_encoder;
7625 NiceMock<MockEncoderSelector> encoder_selector;
7626 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7627 video_send_config_.encoder_settings.encoder_switch_request_callback =
7628 &switch_callback;
7629 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7630 &video_encoder, &encoder_selector);
7631 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7632
7633 // Reset encoder for new configuration to take effect.
7634 ConfigureEncoder(video_encoder_config_.Copy());
7635
7636 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7637 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7638 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007639 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007640 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7641 /*stable_target_bitrate=*/
7642 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7643 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007644 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007645 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007646 /*cwnd_reduce_ratio=*/0);
7647
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007648 ON_CALL(video_encoder, Encode)
philipel9b058032020-02-10 11:30:00 +01007649 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007650 ON_CALL(encoder_selector, OnEncoderBroken)
philipel9b058032020-02-10 11:30:00 +01007651 .WillByDefault(Return(SdpVideoFormat("AV2")));
7652
7653 rtc::Event encode_attempted;
7654 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007655 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7656 /*allow_default_fallback=*/true))
7657 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
philipel9b058032020-02-10 11:30:00 +01007658
7659 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
Markus Handell2cfc1af2022-08-19 08:16:48 +00007660 encode_attempted.Wait(TimeDelta::Seconds(3));
philipel9b058032020-02-10 11:30:00 +01007661
Markus Handell28c71802021-11-08 10:11:55 +01007662 AdvanceTime(TimeDelta::Zero());
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007663
philipel9b058032020-02-10 11:30:00 +01007664 video_stream_encoder_->Stop();
7665
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007666 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7667 // to it's factory, so in order for the encoder instance in the
7668 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7669 // reset the `video_stream_encoder_` here.
7670 video_stream_encoder_.reset();
7671}
7672
7673TEST_F(VideoStreamEncoderTest, SwitchEncoderOnInitFailureWithEncoderSelector) {
7674 NiceMock<MockVideoEncoder> video_encoder;
7675 NiceMock<MockEncoderSelector> encoder_selector;
7676 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7677 video_send_config_.encoder_settings.encoder_switch_request_callback =
7678 &switch_callback;
7679 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7680 &video_encoder, &encoder_selector);
7681 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7682
7683 // Reset encoder for new configuration to take effect.
7684 ConfigureEncoder(video_encoder_config_.Copy());
7685
7686 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7687 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7688 /*round_trip_time_ms=*/0,
7689 /*cwnd_reduce_ratio=*/0);
7690 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7691
7692 ON_CALL(video_encoder, InitEncode(_, _))
7693 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7694 ON_CALL(encoder_selector, OnEncoderBroken)
7695 .WillByDefault(Return(SdpVideoFormat("AV2")));
7696
7697 rtc::Event encode_attempted;
7698 EXPECT_CALL(switch_callback,
7699 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7700 /*allow_default_fallback=*/true))
7701 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7702
7703 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
Markus Handell2cfc1af2022-08-19 08:16:48 +00007704 encode_attempted.Wait(TimeDelta::Seconds(3));
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007705
7706 AdvanceTime(TimeDelta::Zero());
7707
7708 video_stream_encoder_->Stop();
7709
7710 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7711 // to it's factory, so in order for the encoder instance in the
7712 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7713 // reset the `video_stream_encoder_` here.
7714 video_stream_encoder_.reset();
7715}
7716
7717TEST_F(VideoStreamEncoderTest,
7718 SwitchEncoderOnInitFailureWithoutEncoderSelector) {
7719 NiceMock<MockVideoEncoder> video_encoder;
7720 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7721 video_send_config_.encoder_settings.encoder_switch_request_callback =
7722 &switch_callback;
7723 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7724 &video_encoder, /*encoder_selector=*/nullptr);
7725 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7726
7727 // Reset encoder for new configuration to take effect.
7728 ConfigureEncoder(video_encoder_config_.Copy());
7729
7730 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7731 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7732 /*round_trip_time_ms=*/0,
7733 /*cwnd_reduce_ratio=*/0);
7734 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7735
7736 ON_CALL(video_encoder, InitEncode(_, _))
7737 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7738
7739 rtc::Event encode_attempted;
7740 EXPECT_CALL(switch_callback,
7741 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "VP8"),
7742 /*allow_default_fallback=*/true))
7743 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7744
7745 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
Markus Handell2cfc1af2022-08-19 08:16:48 +00007746 encode_attempted.Wait(TimeDelta::Seconds(3));
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007747
7748 AdvanceTime(TimeDelta::Zero());
7749
7750 video_stream_encoder_->Stop();
7751
7752 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007753 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007754 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7755 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007756 video_stream_encoder_.reset();
7757}
7758
Byoungchan Lee13fe3672022-04-06 10:44:42 +09007759TEST_F(VideoStreamEncoderTest, NullEncoderReturnSwitch) {
7760 // As a variant of EncoderSelectorBrokenEncoderSwitch, when a null
7761 // VideoEncoder is passed in encoder_factory, it checks whether
7762 // Codec Switch occurs without a crash.
7763 constexpr int kSufficientBitrateToNotDrop = 1000;
7764 constexpr int kDontCare = 100;
7765
7766 NiceMock<MockEncoderSelector> encoder_selector;
7767 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7768 video_send_config_.encoder_settings.encoder_switch_request_callback =
7769 &switch_callback;
7770 auto encoder_factory =
7771 std::make_unique<test::VideoEncoderNullableProxyFactory>(
7772 /*encoder=*/nullptr, &encoder_selector);
7773 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7774
7775 // Reset encoder for new configuration to take effect.
7776 ConfigureEncoder(video_encoder_config_.Copy());
7777 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7778 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7779 // not fail.
7780 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7781 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7782 /*stable_target_bitrate=*/
7783 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7784 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7785 /*fraction_lost=*/0,
7786 /*round_trip_time_ms=*/0,
7787 /*cwnd_reduce_ratio=*/0);
7788 ON_CALL(encoder_selector, OnEncoderBroken)
7789 .WillByDefault(Return(SdpVideoFormat("AV2")));
7790 rtc::Event encode_attempted;
7791 EXPECT_CALL(switch_callback,
7792 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7793 /*allow_default_fallback=*/_))
7794 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7795
7796 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
Markus Handell2cfc1af2022-08-19 08:16:48 +00007797 encode_attempted.Wait(TimeDelta::Seconds(3));
Byoungchan Lee13fe3672022-04-06 10:44:42 +09007798
7799 AdvanceTime(TimeDelta::Zero());
7800
7801 video_stream_encoder_->Stop();
7802
7803 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7804 // to it's factory, so in order for the encoder instance in the
7805 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7806 // reset the `video_stream_encoder_` here.
7807 video_stream_encoder_.reset();
7808}
7809
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007810TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007811 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007812 const int kFrameWidth = 320;
7813 const int kFrameHeight = 180;
7814
7815 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007816 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007817 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007818 /*target_bitrate=*/rate,
7819 /*stable_target_bitrate=*/rate,
7820 /*link_allocation=*/rate,
7821 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007822 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007823 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007824
7825 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007826 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007827 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7828 frame.set_rotation(kVideoRotation_270);
7829 video_source_.IncomingCapturedFrame(frame);
7830 WaitForEncodedFrame(timestamp_ms);
7831 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7832
7833 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007834 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007835 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007836 /*target_bitrate=*/new_stable_rate,
7837 /*stable_target_bitrate=*/new_stable_rate,
7838 /*link_allocation=*/rate,
7839 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007840 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007841 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007842 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7843 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7844 video_stream_encoder_->Stop();
7845}
7846
7847TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007848 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007849 const int kFrameWidth = 320;
7850 const int kFrameHeight = 180;
7851
7852 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007853 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007854 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007855 /*target_bitrate=*/rate,
7856 /*stable_target_bitrate=*/rate,
7857 /*link_allocation=*/rate,
7858 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007859 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007860 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007861
7862 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007863 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007864 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7865 frame.set_rotation(kVideoRotation_270);
7866 video_source_.IncomingCapturedFrame(frame);
7867 WaitForEncodedFrame(timestamp_ms);
7868 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7869
7870 // Set a higher target rate without changing the link_allocation. Should not
7871 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007872 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007873 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007874 /*target_bitrate=*/rate,
7875 /*stable_target_bitrate=*/new_stable_rate,
7876 /*link_allocation=*/rate,
7877 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007878 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007879 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007880 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7881 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7882 video_stream_encoder_->Stop();
7883}
7884
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007885TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01007886 test::ScopedKeyValueConfig field_trials(
7887 field_trials_,
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007888 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7889 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7890 const int kFramerateFps = 30;
7891 const int kWidth = 1920;
7892 const int kHeight = 1080;
7893 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7894 // Works on screenshare mode.
7895 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7896 // We rely on the automatic resolution adaptation, but we handle framerate
7897 // adaptation manually by mocking the stats proxy.
7898 video_source_.set_adaptation_enabled(true);
7899
7900 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007901 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007902 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007903 video_stream_encoder_->SetSource(&video_source_,
7904 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007905 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007906
7907 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7908 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7909
7910 // Pass enough frames with the full update to trigger animation detection.
7911 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007912 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007913 frame.set_ntp_time_ms(timestamp_ms);
7914 frame.set_timestamp_us(timestamp_ms * 1000);
7915 video_source_.IncomingCapturedFrame(frame);
7916 WaitForEncodedFrame(timestamp_ms);
7917 }
7918
7919 // Resolution should be limited.
7920 rtc::VideoSinkWants expected;
7921 expected.max_framerate_fps = kFramerateFps;
7922 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007923 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007924
7925 // Pass one frame with no known update.
7926 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007927 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007928 frame.set_ntp_time_ms(timestamp_ms);
7929 frame.set_timestamp_us(timestamp_ms * 1000);
7930 frame.clear_update_rect();
7931
7932 video_source_.IncomingCapturedFrame(frame);
7933 WaitForEncodedFrame(timestamp_ms);
7934
7935 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007936 EXPECT_THAT(video_source_.sink_wants(),
7937 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007938
7939 video_stream_encoder_->Stop();
7940}
7941
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007942TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7943 const int kWidth = 720; // 540p adapted down.
7944 const int kHeight = 405;
7945 const int kNumFrames = 3;
7946 // Works on screenshare mode.
7947 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7948 /*num_spatial_layers=*/2, /*screenshare=*/true);
7949
7950 video_source_.set_adaptation_enabled(true);
7951
7952 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007953 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007954
7955 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7956
7957 // Pass enough frames with the full update to trigger animation detection.
7958 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007959 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007960 frame.set_ntp_time_ms(timestamp_ms);
7961 frame.set_timestamp_us(timestamp_ms * 1000);
7962 video_source_.IncomingCapturedFrame(frame);
7963 WaitForEncodedFrame(timestamp_ms);
7964 }
7965
7966 video_stream_encoder_->Stop();
7967}
7968
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007969TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7970 const float downscale_factors[] = {4.0, 2.0, 1.0};
7971 const int number_layers =
7972 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7973 VideoEncoderConfig config;
7974 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7975 for (int i = 0; i < number_layers; ++i) {
7976 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7977 config.simulcast_layers[i].active = true;
7978 }
7979 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007980 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007981 "VP8", /*max qp*/ 56, /*screencast*/ false,
7982 /*screenshare enabled*/ false);
7983 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007984 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7985 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007986
7987 // First initialization.
7988 // Encoder should be initialized. Next frame should be key frame.
7989 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7990 sink_.SetNumExpectedLayers(number_layers);
7991 int64_t timestamp_ms = kFrameIntervalMs;
7992 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7993 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007994 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007995 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7996 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7997 VideoFrameType::kVideoFrameKey,
7998 VideoFrameType::kVideoFrameKey}));
7999
8000 // Disable top layer.
8001 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
8002 config.simulcast_layers[number_layers - 1].active = false;
8003 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8004 sink_.SetNumExpectedLayers(number_layers - 1);
8005 timestamp_ms += kFrameIntervalMs;
8006 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8007 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008008 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008009 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8010 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
8011 VideoFrameType::kVideoFrameDelta,
8012 VideoFrameType::kVideoFrameDelta}));
8013
8014 // Re-enable top layer.
8015 // Encoder should be re-initialized. Next frame should be key frame.
8016 config.simulcast_layers[number_layers - 1].active = true;
8017 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8018 sink_.SetNumExpectedLayers(number_layers);
8019 timestamp_ms += kFrameIntervalMs;
8020 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8021 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008022 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008023 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8024 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
8025 VideoFrameType::kVideoFrameKey,
8026 VideoFrameType::kVideoFrameKey}));
8027
8028 // Top layer max rate change.
8029 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
8030 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
8031 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8032 sink_.SetNumExpectedLayers(number_layers);
8033 timestamp_ms += kFrameIntervalMs;
8034 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8035 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008036 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008037 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8038 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
8039 VideoFrameType::kVideoFrameDelta,
8040 VideoFrameType::kVideoFrameDelta}));
8041
8042 // Top layer resolution change.
8043 // Encoder should be re-initialized. Next frame should be key frame.
8044 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
8045 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8046 sink_.SetNumExpectedLayers(number_layers);
8047 timestamp_ms += kFrameIntervalMs;
8048 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8049 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008050 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008051 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8052 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
8053 VideoFrameType::kVideoFrameKey,
8054 VideoFrameType::kVideoFrameKey}));
8055 video_stream_encoder_->Stop();
8056}
8057
Henrik Boström1124ed12021-02-25 10:30:39 +01008058TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
8059 const int kFrameWidth = 1280;
8060 const int kFrameHeight = 720;
8061
8062 SetUp();
8063 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008064 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01008065
8066 // Capturing a frame should reconfigure the encoder and expose the encoder
8067 // resolution, which is the same as the input frame.
8068 int64_t timestamp_ms = kFrameIntervalMs;
8069 video_source_.IncomingCapturedFrame(
8070 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8071 WaitForEncodedFrame(timestamp_ms);
8072 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8073 EXPECT_THAT(video_source_.sink_wants().resolutions,
8074 ::testing::ElementsAreArray(
8075 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
8076
8077 video_stream_encoder_->Stop();
8078}
8079
8080TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
8081 // Pick downscale factors such that we never encode at full resolution - this
8082 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 16:22:11 +02008083 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 10:30:39 +01008084 // encoder should not ask for the frame resolution. This allows video frames
8085 // to have the appearence of one resolution but optimize its internal buffers
8086 // for what is actually encoded.
8087 const size_t kNumSimulcastLayers = 3u;
8088 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
8089 const int kFrameWidth = 1280;
8090 const int kFrameHeight = 720;
8091 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8092 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8093 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8094 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8095 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8096 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8097
8098 VideoEncoderConfig config;
8099 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
8100 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
8101 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
8102 config.simulcast_layers[i].active = true;
8103 }
8104 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02008105 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01008106 "VP8", /*max qp*/ 56, /*screencast*/ false,
8107 /*screenshare enabled*/ false);
8108 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008109 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8110 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01008111
8112 // Capture a frame with all layers active.
8113 int64_t timestamp_ms = kFrameIntervalMs;
8114 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
8115 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8116 video_source_.IncomingCapturedFrame(
8117 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8118 WaitForEncodedFrame(timestamp_ms);
8119 // Expect encoded resolutions to match the expected simulcast layers.
8120 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8121 EXPECT_THAT(
8122 video_source_.sink_wants().resolutions,
8123 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
8124
8125 // Capture a frame with one of the layers inactive.
8126 timestamp_ms += kFrameIntervalMs;
8127 config.simulcast_layers[2].active = false;
8128 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
8129 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8130 video_source_.IncomingCapturedFrame(
8131 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8132 WaitForEncodedFrame(timestamp_ms);
8133
8134 // Expect encoded resolutions to match the expected simulcast layers.
8135 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8136 EXPECT_THAT(video_source_.sink_wants().resolutions,
8137 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
8138
8139 // Capture a frame with all but one layer turned off.
8140 timestamp_ms += kFrameIntervalMs;
8141 config.simulcast_layers[1].active = false;
8142 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
8143 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8144 video_source_.IncomingCapturedFrame(
8145 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8146 WaitForEncodedFrame(timestamp_ms);
8147
8148 // Expect encoded resolutions to match the expected simulcast layers.
8149 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8150 EXPECT_THAT(video_source_.sink_wants().resolutions,
8151 ::testing::ElementsAreArray({kLayer0Size}));
8152
8153 video_stream_encoder_->Stop();
8154}
8155
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008156TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008157 ResetEncoder("VP8", 1, 1, 1, false);
8158
Niels Möller8b692902021-06-14 12:04:57 +02008159 // Force encoder reconfig.
8160 video_source_.IncomingCapturedFrame(
8161 CreateFrame(1, codec_width_, codec_height_));
8162 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8163
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008164 // Set QP on encoded frame and pass the frame to encode complete callback.
8165 // Since QP is present QP parsing won't be triggered and the original value
8166 // should be kept.
8167 EncodedImage encoded_image;
8168 encoded_image.qp_ = 123;
8169 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8170 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8171 CodecSpecificInfo codec_info;
8172 codec_info.codecType = kVideoCodecVP8;
8173 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
Markus Handell2cfc1af2022-08-19 08:16:48 +00008174 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeout));
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008175 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
8176 video_stream_encoder_->Stop();
8177}
8178
8179TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008180 ResetEncoder("VP8", 1, 1, 1, false);
8181
Niels Möller8b692902021-06-14 12:04:57 +02008182 // Force encoder reconfig.
8183 video_source_.IncomingCapturedFrame(
8184 CreateFrame(1, codec_width_, codec_height_));
8185 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8186
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008187 // Pass an encoded frame without QP to encode complete callback. QP should be
8188 // parsed and set.
8189 EncodedImage encoded_image;
8190 encoded_image.qp_ = -1;
8191 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8192 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8193 CodecSpecificInfo codec_info;
8194 codec_info.codecType = kVideoCodecVP8;
8195 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
Markus Handell2cfc1af2022-08-19 08:16:48 +00008196 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeout));
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008197 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
8198 video_stream_encoder_->Stop();
8199}
8200
8201TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01008202 webrtc::test::ScopedKeyValueConfig field_trials(
8203 field_trials_, "WebRTC-QpParsingKillSwitch/Enabled/");
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008204
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008205 ResetEncoder("VP8", 1, 1, 1, false);
8206
Niels Möller8b692902021-06-14 12:04:57 +02008207 // Force encoder reconfig.
8208 video_source_.IncomingCapturedFrame(
8209 CreateFrame(1, codec_width_, codec_height_));
8210 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8211
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008212 EncodedImage encoded_image;
8213 encoded_image.qp_ = -1;
8214 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8215 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8216 CodecSpecificInfo codec_info;
8217 codec_info.codecType = kVideoCodecVP8;
8218 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
Markus Handell2cfc1af2022-08-19 08:16:48 +00008219 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeout));
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008220 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
8221 video_stream_encoder_->Stop();
8222}
8223
Sergey Silkind19e3b92021-03-16 10:05:30 +00008224TEST_F(VideoStreamEncoderTest,
8225 QualityScalingNotAllowed_QualityScalingDisabled) {
8226 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8227
8228 // Disable scaling settings in encoder info.
8229 fake_encoder_.SetQualityScaling(false);
8230 // Disable quality scaling in encoder config.
8231 video_encoder_config.is_quality_scaling_allowed = false;
8232 ConfigureEncoder(std::move(video_encoder_config));
8233
8234 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008235 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008236
8237 test::FrameForwarder source;
8238 video_stream_encoder_->SetSource(
8239 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8240 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8241 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8242
8243 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8244 WaitForEncodedFrame(1);
8245 video_stream_encoder_->TriggerQualityLow();
8246 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8247
8248 video_stream_encoder_->Stop();
8249}
8250
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008251TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
8252 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8253
8254 // Disable scaling settings in encoder info.
8255 fake_encoder_.SetQualityScaling(false);
8256 // Set QP trusted in encoder info.
8257 fake_encoder_.SetIsQpTrusted(true);
8258 // Enable quality scaling in encoder config.
8259 video_encoder_config.is_quality_scaling_allowed = false;
8260 ConfigureEncoder(std::move(video_encoder_config));
8261
8262 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008263 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008264
8265 test::FrameForwarder source;
8266 video_stream_encoder_->SetSource(
8267 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8268 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8269 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8270
8271 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8272 WaitForEncodedFrame(1);
8273 video_stream_encoder_->TriggerQualityLow();
8274 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8275
8276 video_stream_encoder_->Stop();
8277}
8278
Shuhai Pengf2707702021-09-29 17:19:44 +08008279TEST_F(VideoStreamEncoderTest,
8280 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8281 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8282
8283 // Disable scaling settings in encoder info.
8284 fake_encoder_.SetQualityScaling(false);
8285 // Set QP trusted in encoder info.
8286 fake_encoder_.SetIsQpTrusted(true);
8287 // Enable quality scaling in encoder config.
8288 video_encoder_config.is_quality_scaling_allowed = false;
8289 ConfigureEncoder(std::move(video_encoder_config));
8290
8291 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008292 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008293
8294 test::FrameForwarder source;
8295 video_stream_encoder_->SetSource(
8296 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8297 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8298 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8299
8300 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8301 WaitForEncodedFrame(1);
8302 video_stream_encoder_->TriggerQualityLow();
8303 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8304
8305 video_stream_encoder_->Stop();
8306}
8307
8308TEST_F(VideoStreamEncoderTest,
8309 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8310 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8311
8312 // Disable scaling settings in encoder info.
8313 fake_encoder_.SetQualityScaling(false);
8314 // Set QP trusted in encoder info.
8315 fake_encoder_.SetIsQpTrusted(false);
8316 // Enable quality scaling in encoder config.
8317 video_encoder_config.is_quality_scaling_allowed = false;
8318 ConfigureEncoder(std::move(video_encoder_config));
8319
8320 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008321 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008322
8323 test::FrameForwarder source;
8324 video_stream_encoder_->SetSource(
8325 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8326 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8327 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8328
8329 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8330 WaitForEncodedFrame(1);
8331 video_stream_encoder_->TriggerQualityLow();
8332 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8333
8334 video_stream_encoder_->Stop();
8335}
8336
8337TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8338 // Set QP trusted in encoder info.
8339 fake_encoder_.SetIsQpTrusted(false);
8340
8341 const int MinEncBitrateKbps = 30;
8342 const int MaxEncBitrateKbps = 100;
8343 const int MinStartBitrateKbp = 50;
8344 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8345 /*frame_size_pixels=*/codec_width_ * codec_height_,
8346 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8347 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8348 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8349
8350 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008351 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008352
8353 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8354
8355 VideoEncoderConfig video_encoder_config;
8356 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8357 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8358 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8359 MinEncBitrateKbps * 1000;
8360 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8361 kMaxPayloadLength);
8362
8363 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8364 WaitForEncodedFrame(1);
8365 EXPECT_EQ(
8366 MaxEncBitrateKbps,
8367 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8368 EXPECT_EQ(
8369 MinEncBitrateKbps,
8370 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8371
8372 video_stream_encoder_->Stop();
8373}
8374
8375TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8376 // Set QP trusted in encoder info.
8377 fake_encoder_.SetIsQpTrusted(false);
8378
8379 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8380 EncoderInfoSettings::
8381 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8382 codec_width_ * codec_height_,
8383 EncoderInfoSettings::
8384 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8385 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8386
8387 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8388 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8389 const int TargetEncBitrate = MaxEncBitrate;
8390 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8391 DataRate::BitsPerSec(TargetEncBitrate),
8392 DataRate::BitsPerSec(TargetEncBitrate),
8393 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8394
8395 VideoEncoderConfig video_encoder_config;
8396 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8397 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8398 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8399 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8400 kMaxPayloadLength);
8401
8402 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8403 WaitForEncodedFrame(1);
8404 EXPECT_EQ(
8405 MaxEncBitrate / 1000,
8406 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8407 EXPECT_EQ(
8408 MinEncBitrate / 1000,
8409 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8410
8411 video_stream_encoder_->Stop();
8412}
8413
Erik Språnge4589cb2022-04-06 16:44:30 +02008414TEST_F(VideoStreamEncoderTest, NormalComplexityWithMoreThanTwoCores) {
8415 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8416 /*num_spatial_layers=*/1,
8417 /*screenshare=*/false, /*allocation_callback_type=*/
8418 VideoStreamEncoder::BitrateAllocationCallbackType::
8419 kVideoBitrateAllocationWhenScreenSharing,
8420 /*num_cores=*/3);
8421
8422 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8423 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8424 video_source_.IncomingCapturedFrame(
8425 CreateFrame(1, /*width=*/320, /*height=*/180));
8426 WaitForEncodedFrame(1);
8427 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8428 VideoCodecComplexity::kComplexityNormal);
8429 video_stream_encoder_->Stop();
8430}
8431
8432TEST_F(VideoStreamEncoderTest,
8433 NormalComplexityWhenLowTierOptimizationsAreDisabled) {
8434 webrtc::test::ScopedKeyValueConfig field_trials(
8435 field_trials_, "WebRTC-VP9-LowTierOptimizations/Disabled/");
8436
8437 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8438 /*num_spatial_layers=*/1,
8439 /*screenshare=*/false, /*allocation_callback_type=*/
8440 VideoStreamEncoder::BitrateAllocationCallbackType::
8441 kVideoBitrateAllocationWhenScreenSharing,
8442 /*num_cores=*/2);
8443
8444 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8445 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8446 video_source_.IncomingCapturedFrame(
8447 CreateFrame(1, /*width=*/320, /*height=*/180));
8448 WaitForEncodedFrame(1);
8449 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8450 VideoCodecComplexity::kComplexityNormal);
8451 video_stream_encoder_->Stop();
8452}
8453
8454TEST_F(VideoStreamEncoderTest, LowComplexityWithTwoCores) {
8455 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8456 /*num_spatial_layers=*/1,
8457 /*screenshare=*/false, /*allocation_callback_type=*/
8458 VideoStreamEncoder::BitrateAllocationCallbackType::
8459 kVideoBitrateAllocationWhenScreenSharing,
8460 /*num_cores=*/2);
8461
8462 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8463 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8464 video_source_.IncomingCapturedFrame(
8465 CreateFrame(1, /*width=*/320, /*height=*/180));
8466 WaitForEncodedFrame(1);
8467 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8468 VideoCodecComplexity::kComplexityLow);
8469 video_stream_encoder_->Stop();
8470}
8471
Sergey Silkind19e3b92021-03-16 10:05:30 +00008472#if !defined(WEBRTC_IOS)
8473// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8474// disabled by default on iOS.
8475TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8476 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8477
8478 // Disable scaling settings in encoder info.
8479 fake_encoder_.SetQualityScaling(false);
8480 // Enable quality scaling in encoder config.
8481 video_encoder_config.is_quality_scaling_allowed = true;
8482 ConfigureEncoder(std::move(video_encoder_config));
8483
8484 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008485 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008486
8487 test::FrameForwarder source;
8488 video_stream_encoder_->SetSource(
8489 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8490 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8491 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8492
8493 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8494 WaitForEncodedFrame(1);
8495 video_stream_encoder_->TriggerQualityLow();
8496 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8497
8498 video_stream_encoder_->Stop();
8499}
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008500
8501TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8502 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8503
8504 // Disable scaling settings in encoder info.
8505 fake_encoder_.SetQualityScaling(false);
8506 // Set QP trusted in encoder info.
8507 fake_encoder_.SetIsQpTrusted(true);
8508 // Enable quality scaling in encoder config.
8509 video_encoder_config.is_quality_scaling_allowed = true;
8510 ConfigureEncoder(std::move(video_encoder_config));
8511
8512 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008513 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008514
8515 test::FrameForwarder source;
8516 video_stream_encoder_->SetSource(
8517 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8518 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8519 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8520
8521 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8522 WaitForEncodedFrame(1);
8523 video_stream_encoder_->TriggerQualityLow();
8524 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8525
8526 video_stream_encoder_->Stop();
8527}
Shuhai Pengf2707702021-09-29 17:19:44 +08008528
8529TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8530 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8531
8532 // Disable scaling settings in encoder info.
8533 fake_encoder_.SetQualityScaling(false);
8534 // Set QP not trusted in encoder info.
8535 fake_encoder_.SetIsQpTrusted(false);
8536 // Enable quality scaling in encoder config.
8537 video_encoder_config.is_quality_scaling_allowed = true;
8538 ConfigureEncoder(std::move(video_encoder_config));
8539
8540 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008541 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008542
8543 test::FrameForwarder source;
8544 video_stream_encoder_->SetSource(
8545 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8546 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8547 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8548
8549 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8550 WaitForEncodedFrame(1);
8551 video_stream_encoder_->TriggerQualityLow();
8552 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8553 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8554 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8555
8556 video_stream_encoder_->Stop();
8557}
8558
8559TEST_F(VideoStreamEncoderTest,
8560 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8561 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8562
8563 // Disable scaling settings in encoder info.
8564 fake_encoder_.SetQualityScaling(false);
8565 // Set QP trusted in encoder info.
8566 fake_encoder_.SetIsQpTrusted(true);
8567 // Enable quality scaling in encoder config.
8568 video_encoder_config.is_quality_scaling_allowed = true;
8569 ConfigureEncoder(std::move(video_encoder_config));
8570
8571 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008572 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008573
8574 test::FrameForwarder source;
8575 video_stream_encoder_->SetSource(
8576 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8577 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8578 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8579
8580 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8581 WaitForEncodedFrame(1);
8582 video_stream_encoder_->TriggerQualityLow();
8583 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8584 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8585
8586 video_stream_encoder_->Stop();
8587}
8588
8589TEST_F(VideoStreamEncoderTest,
8590 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8591 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8592
8593 // Disable scaling settings in encoder info.
8594 fake_encoder_.SetQualityScaling(false);
8595 // Set QP trusted in encoder info.
8596 fake_encoder_.SetIsQpTrusted(false);
8597 // Enable quality scaling in encoder config.
8598 video_encoder_config.is_quality_scaling_allowed = true;
8599 ConfigureEncoder(std::move(video_encoder_config));
8600
8601 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008602 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008603
8604 test::FrameForwarder source;
8605 video_stream_encoder_->SetSource(
8606 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8607 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8608 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8609
8610 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8611 WaitForEncodedFrame(1);
8612 video_stream_encoder_->TriggerQualityLow();
8613 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8614
8615 video_stream_encoder_->Stop();
8616}
8617
Erik Språng5e13d052022-08-02 11:42:49 +02008618TEST_F(VideoStreamEncoderTest,
8619 RequestsRefreshFrameAfterEarlyDroppedNativeFrame) {
8620 // Send a native frame before encoder rates have been set. The encoder is
8621 // seen as paused at this time.
8622 rtc::Event frame_destroyed_event;
8623 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
8624 /*ntp_time_ms=*/1, &frame_destroyed_event, codec_width_, codec_height_));
8625
8626 // Frame should be dropped and destroyed.
8627 ExpectDroppedFrame();
Markus Handell2cfc1af2022-08-19 08:16:48 +00008628 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
Erik Språng5e13d052022-08-02 11:42:49 +02008629 EXPECT_EQ(video_source_.refresh_frames_requested_, 0);
8630
8631 // Set bitrates, unpausing the encoder and triggering a request for a refresh
8632 // frame.
8633 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8634 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8635 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8636 EXPECT_EQ(video_source_.refresh_frames_requested_, 1);
8637
8638 video_stream_encoder_->Stop();
8639}
8640
Erik Språnge4589cb2022-04-06 16:44:30 +02008641#endif // !defined(WEBRTC_IOS)
Sergey Silkind19e3b92021-03-16 10:05:30 +00008642
Henrik Boström56db9ff2021-03-24 09:06:45 +01008643// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8644class VideoStreamEncoderWithRealEncoderTest
8645 : public VideoStreamEncoderTest,
8646 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8647 public:
8648 VideoStreamEncoderWithRealEncoderTest()
8649 : VideoStreamEncoderTest(),
8650 codec_type_(std::get<0>(GetParam())),
8651 allow_i420_conversion_(std::get<1>(GetParam())) {}
8652
8653 void SetUp() override {
8654 VideoStreamEncoderTest::SetUp();
8655 std::unique_ptr<VideoEncoder> encoder;
8656 switch (codec_type_) {
8657 case kVideoCodecVP8:
8658 encoder = VP8Encoder::Create();
8659 break;
8660 case kVideoCodecVP9:
8661 encoder = VP9Encoder::Create();
8662 break;
8663 case kVideoCodecAV1:
philipel09a28482022-05-25 09:47:06 +02008664 encoder = CreateLibaomAv1Encoder();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008665 break;
8666 case kVideoCodecH264:
8667 encoder =
8668 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8669 break;
8670 case kVideoCodecMultiplex:
8671 mock_encoder_factory_for_multiplex_ =
8672 std::make_unique<MockVideoEncoderFactory>();
8673 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8674 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8675 .WillRepeatedly([] { return VP8Encoder::Create(); });
8676 encoder = std::make_unique<MultiplexEncoderAdapter>(
8677 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8678 false);
8679 break;
8680 default:
Artem Titovd3251962021-11-15 16:57:07 +01008681 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008682 }
8683 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8684 }
8685
8686 void TearDown() override {
8687 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 16:22:11 +02008688 // Ensure `video_stream_encoder_` is destroyed before
8689 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 09:06:45 +01008690 video_stream_encoder_.reset();
8691 VideoStreamEncoderTest::TearDown();
8692 }
8693
8694 protected:
8695 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8696 std::unique_ptr<VideoEncoder> encoder) {
8697 // Configure VSE to use the encoder.
8698 encoder_ = std::move(encoder);
8699 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8700 encoder_.get(), &encoder_selector_);
8701 video_send_config_.encoder_settings.encoder_factory =
8702 encoder_proxy_factory_.get();
8703 VideoEncoderConfig video_encoder_config;
8704 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8705 video_encoder_config_ = video_encoder_config.Copy();
8706 ConfigureEncoder(video_encoder_config_.Copy());
8707
8708 // Set bitrate to ensure frame is not dropped.
8709 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008710 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008711 }
8712
8713 const VideoCodecType codec_type_;
8714 const bool allow_i420_conversion_;
8715 NiceMock<MockEncoderSelector> encoder_selector_;
8716 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8717 std::unique_ptr<VideoEncoder> encoder_;
8718 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8719};
8720
8721TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8722 auto native_i420_frame = test::CreateMappableNativeFrame(
8723 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8724 video_source_.IncomingCapturedFrame(native_i420_frame);
8725 WaitForEncodedFrame(codec_width_, codec_height_);
8726
8727 auto mappable_native_buffer =
8728 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8729 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8730 mappable_native_buffer->GetMappedFramedBuffers();
8731 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8732 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8733 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8734 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8735}
8736
8737TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8738 auto native_nv12_frame = test::CreateMappableNativeFrame(
8739 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8740 video_source_.IncomingCapturedFrame(native_nv12_frame);
8741 WaitForEncodedFrame(codec_width_, codec_height_);
8742
8743 auto mappable_native_buffer =
8744 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8745 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8746 mappable_native_buffer->GetMappedFramedBuffers();
8747 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8748 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8749 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8750 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8751
8752 if (!allow_i420_conversion_) {
8753 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8754 }
8755}
8756
Erik Språng7444b192021-06-02 14:02:13 +02008757TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8758 if (codec_type_ == kVideoCodecMultiplex) {
8759 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8760 return;
8761 }
8762
8763 const size_t kNumSpatialLayers = 3u;
8764 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8765 const int kFrameWidth = 1280;
8766 const int kFrameHeight = 720;
8767 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8768 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8769 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8770 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8771 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8772 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8773
8774 VideoEncoderConfig config;
8775 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8776 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008777 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008778 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8779 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8780 vp9_settings.numberOfTemporalLayers = 3;
8781 vp9_settings.automaticResizeOn = false;
8782 config.encoder_specific_settings =
8783 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8784 vp9_settings);
8785 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8786 /*fps=*/30.0,
8787 /*first_active_layer=*/0,
8788 /*num_spatial_layers=*/3,
8789 /*num_temporal_layers=*/3,
8790 /*is_screenshare=*/false);
8791 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8792 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008793 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008794 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8795 /*fps=*/30.0,
8796 /*first_active_layer=*/0,
8797 /*num_spatial_layers=*/3,
8798 /*num_temporal_layers=*/3,
8799 /*is_screenshare=*/false);
Niels Möller79d566b2022-04-29 11:03:13 +02008800 config.simulcast_layers[0].scalability_mode = ScalabilityMode::kL3T3_KEY;
Erik Språng7444b192021-06-02 14:02:13 +02008801 } else {
8802 // Simulcast for VP8/H264.
8803 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8804 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8805 config.simulcast_layers[i].scale_resolution_down_by =
8806 kDownscaleFactors[i];
8807 config.simulcast_layers[i].active = true;
8808 }
8809 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8810 // Turn off frame dropping to prevent flakiness.
Niels Möller807328f2022-05-12 16:16:39 +02008811 config.frame_drop_enabled = false;
Erik Språng7444b192021-06-02 14:02:13 +02008812 }
8813 }
8814
8815 auto set_layer_active = [&](int layer_idx, bool active) {
8816 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8817 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8818 config.spatial_layers[layer_idx].active = active;
8819 } else {
8820 config.simulcast_layers[layer_idx].active = active;
8821 }
8822 };
8823
8824 config.video_stream_factory =
8825 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8826 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8827 /*screencast*/ false,
8828 /*screenshare enabled*/ false);
8829 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008830 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8831 0, 0, 0);
Erik Språng7444b192021-06-02 14:02:13 +02008832
8833 // Capture a frame with all layers active.
8834 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8835 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8836 int64_t timestamp_ms = kFrameIntervalMs;
8837 video_source_.IncomingCapturedFrame(
8838 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8839
8840 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8841 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8842
8843 // Capture a frame with one of the layers inactive.
8844 set_layer_active(2, false);
8845 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8846 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8847 timestamp_ms += kFrameIntervalMs;
8848 video_source_.IncomingCapturedFrame(
8849 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8850 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8851
8852 // New target bitrates signaled based on lower resolution.
8853 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8854 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8855 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8856 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8857
8858 // Re-enable the top layer.
8859 set_layer_active(2, true);
8860 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8861 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8862 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8863
8864 // Bitrate target adjusted back up to enable HD layer...
8865 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8866 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8867 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8868 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8869
8870 // ...then add a new frame.
8871 timestamp_ms += kFrameIntervalMs;
8872 video_source_.IncomingCapturedFrame(
8873 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8874 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8875 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8876
8877 video_stream_encoder_->Stop();
8878}
8879
Henrik Boström56db9ff2021-03-24 09:06:45 +01008880std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8881 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8882 VideoCodecType codec_type = std::get<0>(info.param);
8883 bool allow_i420_conversion = std::get<1>(info.param);
8884 std::string str;
8885 switch (codec_type) {
8886 case kVideoCodecGeneric:
8887 str = "Generic";
8888 break;
8889 case kVideoCodecVP8:
8890 str = "VP8";
8891 break;
8892 case kVideoCodecVP9:
8893 str = "VP9";
8894 break;
8895 case kVideoCodecAV1:
8896 str = "AV1";
8897 break;
8898 case kVideoCodecH264:
8899 str = "H264";
8900 break;
8901 case kVideoCodecMultiplex:
8902 str = "Multiplex";
8903 break;
8904 default:
Artem Titovd3251962021-11-15 16:57:07 +01008905 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008906 }
8907 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8908 return str;
8909}
8910
8911constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8912 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8913constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8914 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8915constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
Ilya Nikolaevskiy1bcdafc2022-03-09 16:01:07 +01008916 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/false);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008917constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8918 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8919#if defined(WEBRTC_USE_H264)
8920constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8921 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8922
8923// The windows compiler does not tolerate #if statements inside the
8924// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8925// and without H264).
8926INSTANTIATE_TEST_SUITE_P(
8927 All,
8928 VideoStreamEncoderWithRealEncoderTest,
8929 ::testing::Values(kVP8DisallowConversion,
8930 kVP9DisallowConversion,
8931 kAV1AllowConversion,
8932 kMultiplexDisallowConversion,
8933 kH264AllowConversion),
8934 TestParametersVideoCodecAndAllowI420ConversionToString);
8935#else
8936INSTANTIATE_TEST_SUITE_P(
8937 All,
8938 VideoStreamEncoderWithRealEncoderTest,
8939 ::testing::Values(kVP8DisallowConversion,
8940 kVP9DisallowConversion,
8941 kAV1AllowConversion,
8942 kMultiplexDisallowConversion),
8943 TestParametersVideoCodecAndAllowI420ConversionToString);
8944#endif
8945
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008946class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8947 protected:
8948 void RunTest(const std::vector<VideoStream>& configs,
8949 const int expected_num_init_encode) {
8950 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008951 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008952 InsertFrameAndWaitForEncoded();
8953 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8954 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008955 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8956 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008957
8958 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8959 ConfigureEncoder(configs[1]);
8960 InsertFrameAndWaitForEncoded();
8961 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8962 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008963 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008964 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 10:07:11 +02008965 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008966
8967 video_stream_encoder_->Stop();
8968 }
8969
8970 void ConfigureEncoder(const VideoStream& stream) {
8971 VideoEncoderConfig config;
8972 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8973 config.max_bitrate_bps = stream.max_bitrate_bps;
8974 config.simulcast_layers[0] = stream;
8975 config.video_stream_factory =
8976 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8977 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8978 /*conference_mode=*/false);
8979 video_stream_encoder_->ConfigureEncoder(std::move(config),
8980 kMaxPayloadLength);
8981 }
8982
8983 void OnBitrateUpdated(DataRate bitrate) {
8984 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8985 bitrate, bitrate, bitrate, 0, 0, 0);
8986 }
8987
8988 void InsertFrameAndWaitForEncoded() {
8989 timestamp_ms_ += kFrameIntervalMs;
8990 video_source_.IncomingCapturedFrame(
8991 CreateFrame(timestamp_ms_, kWidth, kHeight));
8992 sink_.WaitForEncodedFrame(timestamp_ms_);
8993 }
8994
8995 void ExpectEqual(const VideoCodec& actual,
8996 const VideoStream& expected) const {
8997 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8998 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8999 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
9000 static_cast<unsigned int>(expected.min_bitrate_bps));
9001 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
9002 static_cast<unsigned int>(expected.max_bitrate_bps));
9003 EXPECT_EQ(actual.simulcastStream[0].width,
9004 kWidth / expected.scale_resolution_down_by);
9005 EXPECT_EQ(actual.simulcastStream[0].height,
9006 kHeight / expected.scale_resolution_down_by);
9007 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
9008 expected.num_temporal_layers);
Niels Möller79d566b2022-04-29 11:03:13 +02009009 EXPECT_EQ(actual.GetScalabilityMode(), expected.scalability_mode);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02009010 }
9011
9012 VideoStream DefaultConfig() const {
9013 VideoStream stream;
9014 stream.max_framerate = 25;
9015 stream.min_bitrate_bps = 35000;
9016 stream.max_bitrate_bps = 900000;
9017 stream.scale_resolution_down_by = 1.0;
9018 stream.num_temporal_layers = 1;
9019 stream.bitrate_priority = 1.0;
Niels Möller79d566b2022-04-29 11:03:13 +02009020 stream.scalability_mode = absl::nullopt;
Åsa Persson4d4f62f2021-09-12 16:14:48 +02009021 return stream;
9022 }
9023
9024 const int kWidth = 640;
9025 const int kHeight = 360;
9026 int64_t timestamp_ms_ = 0;
9027};
9028
9029TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
9030 VideoStream config1 = DefaultConfig();
9031 VideoStream config2 = config1;
9032 config2.max_framerate++;
9033
9034 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9035}
9036
9037TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
9038 VideoStream config1 = DefaultConfig();
9039 VideoStream config2 = config1;
9040 config2.min_bitrate_bps += 10000;
9041
9042 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9043}
9044
9045TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
9046 VideoStream config1 = DefaultConfig();
9047 VideoStream config2 = config1;
9048 config2.max_bitrate_bps += 100000;
9049
9050 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9051}
9052
9053TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
9054 VideoStream config1 = DefaultConfig();
9055 VideoStream config2 = config1;
9056 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
9057
9058 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9059}
9060
9061TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
9062 VideoStream config1 = DefaultConfig();
9063 VideoStream config2 = config1;
9064 config2.scale_resolution_down_by *= 2;
9065
9066 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9067}
9068
9069TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
9070 VideoStream config1 = DefaultConfig();
9071 VideoStream config2 = config1;
9072 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
9073
9074 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9075}
9076
9077TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
9078 VideoStream config1 = DefaultConfig();
9079 VideoStream config2 = config1;
Niels Möller05954892022-05-13 13:34:37 +02009080 config2.scalability_mode = ScalabilityMode::kL2T1;
9081
9082 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9083}
9084
9085TEST_F(ReconfigureEncoderTest,
9086 UpdatesNumTemporalLayersFromScalabilityModeChanges) {
9087 VideoStream config1 = DefaultConfig();
9088 VideoStream config2 = config1;
Niels Möller79d566b2022-04-29 11:03:13 +02009089 config2.scalability_mode = ScalabilityMode::kL1T2;
Niels Möller05954892022-05-13 13:34:37 +02009090 config2.num_temporal_layers = 2;
Åsa Persson4d4f62f2021-09-12 16:14:48 +02009091
9092 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9093}
9094
Tommi62b01db2022-01-25 23:41:22 +01009095// Simple test that just creates and then immediately destroys an encoder.
9096// The purpose of the test is to make sure that nothing bad happens if the
9097// initialization step on the encoder queue, doesn't run.
9098TEST(VideoStreamEncoderSimpleTest, CreateDestroy) {
9099 class SuperLazyTaskQueue : public webrtc::TaskQueueBase {
9100 public:
9101 SuperLazyTaskQueue() = default;
9102 ~SuperLazyTaskQueue() override = default;
9103
9104 private:
9105 void Delete() override { delete this; }
Danil Chapovalov95eeaa72022-07-06 10:14:29 +02009106 void PostTask(absl::AnyInvocable<void() &&> task) override {
Tommi62b01db2022-01-25 23:41:22 +01009107 // meh.
9108 }
Danil Chapovalov95eeaa72022-07-06 10:14:29 +02009109 void PostDelayedTask(absl::AnyInvocable<void() &&> task,
9110 TimeDelta delay) override {
Tommi62b01db2022-01-25 23:41:22 +01009111 ASSERT_TRUE(false);
9112 }
Danil Chapovalov95eeaa72022-07-06 10:14:29 +02009113 void PostDelayedHighPrecisionTask(absl::AnyInvocable<void() &&> task,
9114 TimeDelta delay) override {
9115 ADD_FAILURE();
9116 }
Tommi62b01db2022-01-25 23:41:22 +01009117 };
9118
9119 // Lots of boiler plate.
Jonas Oreland8ca06132022-03-14 12:52:48 +01009120 test::ScopedKeyValueConfig field_trials;
Philipp Hanckea204ad22022-07-08 18:43:25 +02009121 GlobalSimulatedTimeController time_controller(Timestamp::Zero());
Tommi62b01db2022-01-25 23:41:22 +01009122 auto stats_proxy = std::make_unique<MockableSendStatisticsProxy>(
9123 time_controller.GetClock(), VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 12:52:48 +01009124 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo, field_trials);
Tommi62b01db2022-01-25 23:41:22 +01009125 SimpleVideoStreamEncoderFactory::MockFakeEncoder mock_fake_encoder(
9126 time_controller.GetClock());
9127 test::VideoEncoderProxyFactory encoder_factory(&mock_fake_encoder);
9128 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory =
9129 CreateBuiltinVideoBitrateAllocatorFactory();
9130 VideoStreamEncoderSettings encoder_settings{
9131 VideoEncoder::Capabilities(/*loss_notification=*/false)};
9132 encoder_settings.encoder_factory = &encoder_factory;
9133 encoder_settings.bitrate_allocator_factory = bitrate_allocator_factory.get();
9134
9135 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9136 EXPECT_CALL((*adapter.get()), Initialize).WillOnce(Return());
9137
9138 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
9139 encoder_queue(new SuperLazyTaskQueue());
9140
9141 // Construct a VideoStreamEncoder instance and let it go out of scope without
9142 // doing anything else (including calling Stop()). This should be fine since
9143 // the posted init task will simply be deleted.
9144 auto encoder = std::make_unique<VideoStreamEncoder>(
9145 time_controller.GetClock(), 1, stats_proxy.get(), encoder_settings,
Markus Handell8e4197b2022-05-30 15:45:28 +02009146 std::make_unique<CpuOveruseDetectorProxy>(stats_proxy.get(),
9147 field_trials),
Tommi62b01db2022-01-25 23:41:22 +01009148 std::move(adapter), std::move(encoder_queue),
9149 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009150 kVideoBitrateAllocation,
9151 field_trials);
Tommi4d6dd172022-05-23 09:34:41 +02009152
9153 // Stop the encoder explicitly. This additional step tests if we could
9154 // hang when calling stop and the TQ has been stopped and/or isn't accepting
9155 // any more tasks.
9156 encoder->Stop();
Tommi62b01db2022-01-25 23:41:22 +01009157}
9158
Markus Handellb4e96d42021-11-05 12:00:55 +01009159TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
9160 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9161 auto* adapter_ptr = adapter.get();
9162 SimpleVideoStreamEncoderFactory factory;
Markus Handell8d87c462021-12-16 11:37:16 +01009163 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9164 nullptr;
9165 EXPECT_CALL(*adapter_ptr, Initialize)
9166 .WillOnce(Invoke([&video_stream_encoder_callback](
9167 FrameCadenceAdapterInterface::Callback* callback) {
9168 video_stream_encoder_callback = callback;
9169 }));
9170 TaskQueueBase* encoder_queue = nullptr;
9171 auto video_stream_encoder =
9172 factory.Create(std::move(adapter), &encoder_queue);
Markus Handellb4e96d42021-11-05 12:00:55 +01009173
Markus Handelle59fee82021-12-23 09:29:23 +01009174 // First a call before we know the frame size and hence cannot compute the
9175 // number of simulcast layers.
9176 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
9177 &FrameCadenceAdapterInterface::
9178 ZeroHertzModeParams::num_simulcast_layers,
Markus Handell5a77e512022-09-01 12:51:50 +00009179 Eq(0u)))));
Markus Handellb4e96d42021-11-05 12:00:55 +01009180 VideoEncoderConfig config;
Markus Handell8d87c462021-12-16 11:37:16 +01009181 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
Markus Handellb4e96d42021-11-05 12:00:55 +01009182 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9183 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
Markus Handelle59fee82021-12-23 09:29:23 +01009184 factory.DepleteTaskQueues();
9185
9186 // Then a call as we've computed the number of simulcast layers after a passed
9187 // frame.
9188 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
9189 &FrameCadenceAdapterInterface::
9190 ZeroHertzModeParams::num_simulcast_layers,
Markus Handell5a77e512022-09-01 12:51:50 +00009191 Gt(0u)))));
Markus Handell8d87c462021-12-16 11:37:16 +01009192 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handell9a478b52021-11-18 16:07:01 +01009193 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01009194 Mock::VerifyAndClearExpectations(adapter_ptr);
9195
Markus Handelle59fee82021-12-23 09:29:23 +01009196 // Expect a disabled zero-hertz mode after passing realtime video.
Markus Handell8d87c462021-12-16 11:37:16 +01009197 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Eq(absl::nullopt)));
Markus Handellb4e96d42021-11-05 12:00:55 +01009198 VideoEncoderConfig config2;
Markus Handell8d87c462021-12-16 11:37:16 +01009199 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config2);
Markus Handellb4e96d42021-11-05 12:00:55 +01009200 config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
9201 video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
Markus Handell8d87c462021-12-16 11:37:16 +01009202 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
Markus Handell9a478b52021-11-18 16:07:01 +01009203 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01009204}
9205
9206TEST(VideoStreamEncoderFrameCadenceTest,
9207 ForwardsFramesIntoFrameCadenceAdapter) {
9208 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9209 auto* adapter_ptr = adapter.get();
9210 test::FrameForwarder video_source;
9211 SimpleVideoStreamEncoderFactory factory;
9212 auto video_stream_encoder = factory.Create(std::move(adapter));
9213 video_stream_encoder->SetSource(
9214 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9215
9216 EXPECT_CALL(*adapter_ptr, OnFrame);
9217 auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
9218 video_source.IncomingCapturedFrame(
Markus Handell8d87c462021-12-16 11:37:16 +01009219 VideoFrame::Builder().set_video_frame_buffer(std::move(buffer)).build());
Markus Handellb4e96d42021-11-05 12:00:55 +01009220}
9221
Markus Handellee225432021-11-29 12:35:12 +01009222TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) {
9223 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9224 auto* adapter_ptr = adapter.get();
9225 test::FrameForwarder video_source;
9226 SimpleVideoStreamEncoderFactory factory;
9227 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9228 nullptr;
9229 EXPECT_CALL(*adapter_ptr, Initialize)
9230 .WillOnce(Invoke([&video_stream_encoder_callback](
9231 FrameCadenceAdapterInterface::Callback* callback) {
9232 video_stream_encoder_callback = callback;
9233 }));
9234 TaskQueueBase* encoder_queue = nullptr;
9235 auto video_stream_encoder =
9236 factory.Create(std::move(adapter), &encoder_queue);
9237
9238 // This is just to make the VSE operational. We'll feed a frame directly by
9239 // the callback interface.
9240 video_stream_encoder->SetSource(
9241 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9242
9243 VideoEncoderConfig video_encoder_config;
9244 test::FillEncoderConfiguration(kVideoCodecGeneric, 1, &video_encoder_config);
9245 video_stream_encoder->ConfigureEncoder(std::move(video_encoder_config),
9246 /*max_data_payload_length=*/1000);
9247
9248 EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps);
9249 EXPECT_CALL(*adapter_ptr, UpdateFrameRate);
Markus Handell8d87c462021-12-16 11:37:16 +01009250 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handellee225432021-11-29 12:35:12 +01009251 factory.DepleteTaskQueues();
9252}
9253
Markus Handell8d87c462021-12-16 11:37:16 +01009254TEST(VideoStreamEncoderFrameCadenceTest,
9255 DeactivatesActivatesLayersOnBitrateChanges) {
9256 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9257 auto* adapter_ptr = adapter.get();
9258 SimpleVideoStreamEncoderFactory factory;
9259 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9260 nullptr;
9261 EXPECT_CALL(*adapter_ptr, Initialize)
9262 .WillOnce(Invoke([&video_stream_encoder_callback](
9263 FrameCadenceAdapterInterface::Callback* callback) {
9264 video_stream_encoder_callback = callback;
9265 }));
9266 TaskQueueBase* encoder_queue = nullptr;
9267 auto video_stream_encoder =
9268 factory.Create(std::move(adapter), &encoder_queue);
9269
9270 // Configure 2 simulcast layers. FillEncoderConfiguration sets min bitrates to
9271 // {150000, 450000}.
9272 VideoEncoderConfig video_encoder_config;
9273 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9274 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9275 kMaxPayloadLength);
9276 // Ensure an encoder is created.
9277 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9278
9279 // Both layers enabled at 1 MBit/s.
9280 video_stream_encoder->OnBitrateUpdated(
9281 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9282 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9283 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9284 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9285 factory.DepleteTaskQueues();
9286 Mock::VerifyAndClearExpectations(adapter_ptr);
9287
9288 // Layer 1 disabled at 200 KBit/s.
9289 video_stream_encoder->OnBitrateUpdated(
9290 DataRate::KilobitsPerSec(200), DataRate::KilobitsPerSec(200),
9291 DataRate::KilobitsPerSec(200), 0, 0, 0);
9292 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9293 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9294 factory.DepleteTaskQueues();
9295 Mock::VerifyAndClearExpectations(adapter_ptr);
9296
9297 // All layers off at suspended video.
9298 video_stream_encoder->OnBitrateUpdated(DataRate::Zero(), DataRate::Zero(),
9299 DataRate::Zero(), 0, 0, 0);
9300 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/false));
9301 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9302 factory.DepleteTaskQueues();
9303 Mock::VerifyAndClearExpectations(adapter_ptr);
9304
9305 // Both layers enabled again back at 1 MBit/s.
9306 video_stream_encoder->OnBitrateUpdated(
9307 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9308 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9309 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9310 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9311 factory.DepleteTaskQueues();
9312}
9313
9314TEST(VideoStreamEncoderFrameCadenceTest, UpdatesQualityConvergence) {
9315 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9316 auto* adapter_ptr = adapter.get();
9317 SimpleVideoStreamEncoderFactory factory;
9318 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9319 nullptr;
9320 EXPECT_CALL(*adapter_ptr, Initialize)
9321 .WillOnce(Invoke([&video_stream_encoder_callback](
9322 FrameCadenceAdapterInterface::Callback* callback) {
9323 video_stream_encoder_callback = callback;
9324 }));
9325 TaskQueueBase* encoder_queue = nullptr;
9326 auto video_stream_encoder =
9327 factory.Create(std::move(adapter), &encoder_queue);
9328
9329 // Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder.
9330 VideoEncoderConfig video_encoder_config;
9331 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9332 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9333 kMaxPayloadLength);
9334 video_stream_encoder->OnBitrateUpdated(
9335 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9336 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9337
9338 // Pass a frame which has unconverged results.
9339 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9340 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9341 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9342 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
Markus Handelle1a198b2022-09-15 08:13:25 +00009343 encoded_image.qp_ = kVp8SteadyStateQpThreshold + 1;
Markus Handell8d87c462021-12-16 11:37:16 +01009344 CodecSpecificInfo codec_specific;
Markus Handelle1a198b2022-09-15 08:13:25 +00009345 codec_specific.codecType = kVideoCodecVP8;
Markus Handell8d87c462021-12-16 11:37:16 +01009346 return codec_specific;
9347 }));
9348 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, false));
9349 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9350 factory.DepleteTaskQueues();
9351 Mock::VerifyAndClearExpectations(adapter_ptr);
9352 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9353
9354 // Pass a frame which converges in layer 0 and not in layer 1.
9355 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9356 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9357 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9358 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
Markus Handelle1a198b2022-09-15 08:13:25 +00009359 // This sets spatial index 0 content to be at target quality, while
9360 // index 1 content is not.
9361 encoded_image.qp_ = kVp8SteadyStateQpThreshold +
9362 (encoded_image.SpatialIndex() == 0 ? 0 : 1);
Markus Handell8d87c462021-12-16 11:37:16 +01009363 CodecSpecificInfo codec_specific;
Markus Handelle1a198b2022-09-15 08:13:25 +00009364 codec_specific.codecType = kVideoCodecVP8;
Markus Handell8d87c462021-12-16 11:37:16 +01009365 return codec_specific;
9366 }));
9367 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, true));
9368 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9369 factory.DepleteTaskQueues();
9370 Mock::VerifyAndClearExpectations(adapter_ptr);
9371 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9372}
9373
Markus Handell2e0f4f02021-12-21 19:14:58 +01009374TEST(VideoStreamEncoderFrameCadenceTest,
9375 RequestsRefreshFramesWhenCadenceAdapterInstructs) {
9376 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9377 auto* adapter_ptr = adapter.get();
9378 MockVideoSourceInterface mock_source;
9379 SimpleVideoStreamEncoderFactory factory;
9380 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9381 nullptr;
9382 EXPECT_CALL(*adapter_ptr, Initialize)
9383 .WillOnce(Invoke([&video_stream_encoder_callback](
9384 FrameCadenceAdapterInterface::Callback* callback) {
9385 video_stream_encoder_callback = callback;
9386 }));
9387 TaskQueueBase* encoder_queue = nullptr;
9388 auto video_stream_encoder =
9389 factory.Create(std::move(adapter), &encoder_queue);
9390 video_stream_encoder->SetSource(
9391 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9392 VideoEncoderConfig config;
9393 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9394 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9395 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9396 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9397 // Ensure the encoder is set up.
9398 factory.DepleteTaskQueues();
9399
Markus Handell818e7fb2021-12-30 13:01:33 +01009400 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest)
9401 .WillOnce(Invoke([video_stream_encoder_callback] {
9402 video_stream_encoder_callback->RequestRefreshFrame();
9403 }));
Markus Handell2e0f4f02021-12-21 19:14:58 +01009404 EXPECT_CALL(mock_source, RequestRefreshFrame);
9405 video_stream_encoder->SendKeyFrame();
9406 factory.DepleteTaskQueues();
9407 Mock::VerifyAndClearExpectations(adapter_ptr);
9408 Mock::VerifyAndClearExpectations(&mock_source);
9409
Markus Handell818e7fb2021-12-30 13:01:33 +01009410 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest);
Markus Handell2e0f4f02021-12-21 19:14:58 +01009411 EXPECT_CALL(mock_source, RequestRefreshFrame).Times(0);
9412 video_stream_encoder->SendKeyFrame();
9413 factory.DepleteTaskQueues();
9414}
9415
Markus Handell818e7fb2021-12-30 13:01:33 +01009416TEST(VideoStreamEncoderFrameCadenceTest,
9417 RequestsRefreshFrameForEarlyZeroHertzKeyFrameRequest) {
9418 SimpleVideoStreamEncoderFactory factory;
9419 auto encoder_queue =
9420 factory.GetTimeController()->GetTaskQueueFactory()->CreateTaskQueue(
9421 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
9422
9423 // Enables zero-hertz mode.
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009424 test::ScopedKeyValueConfig field_trials(
9425 "WebRTC-ZeroHertzScreenshare/Enabled/");
Markus Handell818e7fb2021-12-30 13:01:33 +01009426 auto adapter = FrameCadenceAdapterInterface::Create(
Jonas Oreland8ca06132022-03-14 12:52:48 +01009427 factory.GetTimeController()->GetClock(), encoder_queue.get(),
9428 field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009429 FrameCadenceAdapterInterface* adapter_ptr = adapter.get();
9430
9431 MockVideoSourceInterface mock_source;
9432 auto video_stream_encoder = factory.CreateWithEncoderQueue(
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009433 std::move(adapter), std::move(encoder_queue), &field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009434
9435 video_stream_encoder->SetSource(
9436 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9437 VideoEncoderConfig config;
9438 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9439 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9440 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9441
9442 // Eventually expect a refresh frame request when requesting a key frame
9443 // before initializing zero-hertz mode. This can happen in reality because the
9444 // threads invoking key frame requests and constraints setup aren't
9445 // synchronized.
9446 EXPECT_CALL(mock_source, RequestRefreshFrame);
9447 video_stream_encoder->SendKeyFrame();
Markus Handellf5a50792022-06-16 14:25:15 +02009448 constexpr int kMaxFps = 30;
9449 adapter_ptr->OnConstraintsChanged(VideoTrackSourceConstraints{0, kMaxFps});
9450 factory.GetTimeController()->AdvanceTime(
9451 TimeDelta::Seconds(1) *
9452 FrameCadenceAdapterInterface::kOnDiscardedFrameRefreshFramePeriod /
9453 kMaxFps);
Markus Handell818e7fb2021-12-30 13:01:33 +01009454}
9455
perkj26091b12016-09-01 01:17:40 -07009456} // namespace webrtc