blob: 1f328ccf81d7a501330324fa534e8e3fca8c2114 [file] [log] [blame]
Jonas Oreland1262eb52022-09-27 16:53:04 +02001
perkj26091b12016-09-01 01:17:40 -07002/*
3 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
4 *
5 * Use of this source code is governed by a BSD-style license
6 * that can be found in the LICENSE file in the root of the source
7 * tree. An additional intellectual property rights grant can be found
8 * in the file PATENTS. All contributing project authors may
9 * be found in the AUTHORS file in the root of the source tree.
10 */
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Markus Handell8e4197b2022-05-30 15:45:28 +020020#include "api/field_trials_view.h"
Markus Handell9a478b52021-11-18 16:07:01 +010021#include "api/rtp_parameters.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020022#include "api/task_queue/default_task_queue_factory.h"
Markus Handell818e7fb2021-12-30 13:01:33 +010023#include "api/task_queue/task_queue_base.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010024#include "api/task_queue/task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020025#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010026#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010027#include "api/test/mock_video_encoder_factory.h"
Markus Handell8d87c462021-12-16 11:37:16 +010028#include "api/units/data_rate.h"
Markus Handellf5a50792022-06-16 14:25:15 +020029#include "api/units/time_delta.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080030#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020031#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020032#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020033#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010034#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010035#include "api/video_codecs/sdp_video_format.h"
Erik Språnge4589cb2022-04-06 16:44:30 +020036#include "api/video_codecs/video_codec.h"
Elad Alon370f93a2019-06-11 14:57:57 +020037#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020038#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010039#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020040#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010041#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020042#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070043#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080044#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020045#include "media/engine/webrtc_video_engine.h"
philipel09a28482022-05-25 09:47:06 +020046#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010047#include "modules/video_coding/codecs/h264/include/h264.h"
48#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
49#include "modules/video_coding/codecs/vp8/include/vp8.h"
50#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020051#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020052#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020053#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010054#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Markus Handelle1a198b2022-09-15 08:13:25 +000055#include "modules/video_coding/utility/vp8_constants.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020056#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010057#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020058#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020059#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080060#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020061#include "rtc_base/synchronization/mutex.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020062#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020063#include "test/encoder_settings.h"
64#include "test/fake_encoder.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010065#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020066#include "test/gmock.h"
67#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010068#include "test/mappable_native_buffer.h"
Jonas Orelandc7f691a2022-03-09 15:12:07 +010069#include "test/scoped_key_value_config.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020070#include "test/time_controller/simulated_time_controller.h"
Byoungchan Lee13fe3672022-04-06 10:44:42 +090071#include "test/video_encoder_nullable_proxy_factory.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020072#include "test/video_encoder_proxy_factory.h"
Jonas Oreland1262eb52022-09-27 16:53:04 +020073#include "video/config/encoder_stream_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010074#include "video/frame_cadence_adapter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020075#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070076
77namespace webrtc {
78
sprang57c2fff2017-01-16 06:24:02 -080079using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020080using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020081using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020082using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020083using ::testing::Ge;
84using ::testing::Gt;
Markus Handellee225432021-11-29 12:35:12 +010085using ::testing::Invoke;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020086using ::testing::Le;
87using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010088using ::testing::Matcher;
Markus Handellb4e96d42021-11-05 12:00:55 +010089using ::testing::Mock;
philipel9b058032020-02-10 11:30:00 +010090using ::testing::NiceMock;
Markus Handellb4e96d42021-11-05 12:00:55 +010091using ::testing::Optional;
philipel9b058032020-02-10 11:30:00 +010092using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010093using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020094using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080095
perkj803d97f2016-11-01 11:45:46 -070096namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020097const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010098const int kQpLow = 1;
99const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200100const int kMinFramerateFps = 2;
101const int kMinBalancedFramerateFps = 7;
Markus Handell2cfc1af2022-08-19 08:16:48 +0000102constexpr TimeDelta kFrameTimeout = TimeDelta::Millis(100);
asapersson5f7226f2016-11-25 04:37:00 -0800103const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 10:07:11 +0200104const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
105const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
106const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
107const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 07:02:22 -0800108const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -0700109const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200110const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +0200111const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200112const VideoEncoder::ResolutionBitrateLimits
113 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
114const VideoEncoder::ResolutionBitrateLimits
115 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800116
Asa Persson606d3cb2021-10-04 10:07:11 +0200117uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200118 0x00, 0x00, 0x03, 0x03, 0xF4,
119 0x05, 0x03, 0xC7, 0xE0, 0x1B,
120 0x41, 0x10, 0x8D, 0x00};
121
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100122const uint8_t kCodedFrameVp8Qp25[] = {
123 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
124 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
125 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
126
Markus Handell818e7fb2021-12-30 13:01:33 +0100127VideoFrame CreateSimpleNV12Frame() {
128 return VideoFrame::Builder()
129 .set_video_frame_buffer(rtc::make_ref_counted<NV12Buffer>(
130 /*width=*/16, /*height=*/16))
131 .build();
132}
133
Markus Handell8d87c462021-12-16 11:37:16 +0100134void PassAFrame(
135 TaskQueueBase* encoder_queue,
136 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback,
137 int64_t ntp_time_ms) {
Danil Chapovalov95eeaa72022-07-06 10:14:29 +0200138 encoder_queue->PostTask([video_stream_encoder_callback, ntp_time_ms] {
139 video_stream_encoder_callback->OnFrame(Timestamp::Millis(ntp_time_ms), 1,
140 CreateSimpleNV12Frame());
141 });
Markus Handell8d87c462021-12-16 11:37:16 +0100142}
143
perkj803d97f2016-11-01 11:45:46 -0700144class TestBuffer : public webrtc::I420Buffer {
145 public:
146 TestBuffer(rtc::Event* event, int width, int height)
147 : I420Buffer(width, height), event_(event) {}
148
149 private:
150 friend class rtc::RefCountedObject<TestBuffer>;
151 ~TestBuffer() override {
152 if (event_)
153 event_->Set();
154 }
155 rtc::Event* const event_;
156};
157
Henrik Boström56db9ff2021-03-24 09:06:45 +0100158// A fake native buffer that can't be converted to I420. Upon scaling, it
159// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700160class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
161 public:
162 FakeNativeBuffer(rtc::Event* event, int width, int height)
163 : event_(event), width_(width), height_(height) {}
164 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
165 int width() const override { return width_; }
166 int height() const override { return height_; }
167 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
168 return nullptr;
169 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100170 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
171 int offset_x,
172 int offset_y,
173 int crop_width,
174 int crop_height,
175 int scaled_width,
176 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200177 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
178 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100179 }
Noah Richards51db4212019-06-12 06:59:12 -0700180
181 private:
182 friend class rtc::RefCountedObject<FakeNativeBuffer>;
183 ~FakeNativeBuffer() override {
184 if (event_)
185 event_->Set();
186 }
187 rtc::Event* const event_;
188 const int width_;
189 const int height_;
190};
191
Evan Shrubsole895556e2020-10-05 09:15:13 +0200192// A fake native buffer that is backed by an NV12 buffer.
193class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
194 public:
195 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
196 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
197
198 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
199 int width() const override { return nv12_buffer_->width(); }
200 int height() const override { return nv12_buffer_->height(); }
201 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
202 return nv12_buffer_->ToI420();
203 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200204 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
205 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
206 if (absl::c_find(types, Type::kNV12) != types.end()) {
207 return nv12_buffer_;
208 }
209 return nullptr;
210 }
Niels Möllerba2de582022-04-20 16:46:26 +0200211 const NV12BufferInterface* GetNV12() const { return nv12_buffer_.get(); }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200212
213 private:
214 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
215 ~FakeNV12NativeBuffer() override {
216 if (event_)
217 event_->Set();
218 }
219 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
220 rtc::Event* const event_;
221};
222
Niels Möller7dc26b72017-12-06 10:27:48 +0100223class CpuOveruseDetectorProxy : public OveruseFrameDetector {
224 public:
Markus Handell8e4197b2022-05-30 15:45:28 +0200225 CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer,
226 const FieldTrialsView& field_trials)
227 : OveruseFrameDetector(metrics_observer, field_trials),
Henrik Boström381d1092020-05-12 18:49:07 +0200228 last_target_framerate_fps_(-1),
229 framerate_updated_event_(true /* manual_reset */,
230 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100231 virtual ~CpuOveruseDetectorProxy() {}
232
233 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200234 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100235 last_target_framerate_fps_ = framerate_fps;
236 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200237 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100238 }
239
240 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200241 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100242 return last_target_framerate_fps_;
243 }
244
Niels Möller4db138e2018-04-19 09:04:13 +0200245 CpuOveruseOptions GetOptions() { return options_; }
246
Henrik Boström381d1092020-05-12 18:49:07 +0200247 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
248
Niels Möller7dc26b72017-12-06 10:27:48 +0100249 private:
Markus Handella3765182020-07-08 13:13:32 +0200250 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100251 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200252 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100253};
254
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200255class FakeVideoSourceRestrictionsListener
256 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200257 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200258 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200259 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200260 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200261 RTC_DCHECK(was_restrictions_updated_);
262 }
263
264 rtc::Event* restrictions_updated_event() {
265 return &restrictions_updated_event_;
266 }
267
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200268 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200269 void OnVideoSourceRestrictionsUpdated(
270 VideoSourceRestrictions restrictions,
271 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200272 rtc::scoped_refptr<Resource> reason,
273 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200274 was_restrictions_updated_ = true;
275 restrictions_updated_event_.Set();
276 }
277
278 private:
279 bool was_restrictions_updated_;
280 rtc::Event restrictions_updated_event_;
281};
282
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200283auto WantsFps(Matcher<int> fps_matcher) {
284 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
285 fps_matcher);
286}
287
288auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
289 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
290 AllOf(max_pixel_matcher, Gt(0)));
291}
292
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200293auto ResolutionMax() {
294 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200295 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200296 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
297 Eq(absl::nullopt)));
298}
299
300auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200301 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200302}
303
304auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200305 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200306}
307
308auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200309 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200310}
311
312auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200313 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200314}
315
316auto FpsMaxResolutionMax() {
317 return AllOf(FpsMax(), ResolutionMax());
318}
319
320auto UnlimitedSinkWants() {
321 return AllOf(FpsUnlimited(), ResolutionMax());
322}
323
324auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
325 Matcher<int> fps_range_matcher;
326
327 if (last_frame_pixels <= 320 * 240) {
328 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200329 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200330 fps_range_matcher = AllOf(Ge(10), Le(15));
331 } else if (last_frame_pixels <= 640 * 480) {
332 fps_range_matcher = Ge(15);
333 } else {
334 fps_range_matcher = Eq(kDefaultFramerate);
335 }
336 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
337 fps_range_matcher);
338}
339
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200340auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
341 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
342 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
343}
344
345auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
346 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
347}
348
349auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
350 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
351}
352
353auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
354 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
355 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
356}
357
358auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
359 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
360 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
361}
362
363auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
364 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
365 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
366}
367
368auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
369 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
370 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
371}
372
mflodmancc3d4422017-08-03 08:27:51 -0700373class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700374 public:
Markus Handell9a478b52021-11-18 16:07:01 +0100375 VideoStreamEncoderUnderTest(
376 TimeController* time_controller,
377 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter,
378 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
379 encoder_queue,
380 SendStatisticsProxy* stats_proxy,
381 const VideoStreamEncoderSettings& settings,
382 VideoStreamEncoder::BitrateAllocationCallbackType
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100383 allocation_callback_type,
Erik Språnge4589cb2022-04-06 16:44:30 +0200384 const FieldTrialsView& field_trials,
385 int num_cores)
Markus Handell8e4197b2022-05-30 15:45:28 +0200386 : VideoStreamEncoder(
387 time_controller->GetClock(),
388 num_cores,
389 stats_proxy,
390 settings,
391 std::unique_ptr<OveruseFrameDetector>(
392 overuse_detector_proxy_ =
393 new CpuOveruseDetectorProxy(stats_proxy, field_trials)),
394 std::move(cadence_adapter),
395 std::move(encoder_queue),
396 allocation_callback_type,
397 field_trials),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200398 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200399 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200400 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200401 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200402 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200403 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200404 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200405 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100406 }
perkj803d97f2016-11-01 11:45:46 -0700407
Henrik Boström381d1092020-05-12 18:49:07 +0200408 void SetSourceAndWaitForRestrictionsUpdated(
409 rtc::VideoSourceInterface<VideoFrame>* source,
410 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200411 FakeVideoSourceRestrictionsListener listener;
412 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200413 SetSource(source, degradation_preference);
Markus Handell2cfc1af2022-08-19 08:16:48 +0000414 listener.restrictions_updated_event()->Wait(TimeDelta::Seconds(5));
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200415 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200416 }
417
418 void SetSourceAndWaitForFramerateUpdated(
419 rtc::VideoSourceInterface<VideoFrame>* source,
420 const DegradationPreference& degradation_preference) {
421 overuse_detector_proxy_->framerate_updated_event()->Reset();
422 SetSource(source, degradation_preference);
Markus Handell2cfc1af2022-08-19 08:16:48 +0000423 overuse_detector_proxy_->framerate_updated_event()->Wait(
424 TimeDelta::Seconds(5));
Henrik Boström381d1092020-05-12 18:49:07 +0200425 }
426
427 void OnBitrateUpdatedAndWaitForManagedResources(
428 DataRate target_bitrate,
429 DataRate stable_target_bitrate,
430 DataRate link_allocation,
431 uint8_t fraction_lost,
432 int64_t round_trip_time_ms,
433 double cwnd_reduce_ratio) {
434 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
435 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
436 // Bitrate is updated on the encoder queue.
437 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200438 }
439
kthelgason2fc52542017-03-03 00:24:41 -0800440 // This is used as a synchronisation mechanism, to make sure that the
441 // encoder queue is not blocked before we start sending it frames.
442 void WaitUntilTaskQueueIsIdle() {
Markus Handell28c71802021-11-08 10:11:55 +0100443 time_controller_->AdvanceTime(TimeDelta::Zero());
kthelgason2fc52542017-03-03 00:24:41 -0800444 }
445
Henrik Boström91aa7322020-04-28 12:24:33 +0200446 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200447 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200448 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200449 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200450 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200451 event.Set();
452 });
Markus Handell2cfc1af2022-08-19 08:16:48 +0000453 ASSERT_TRUE(event.Wait(TimeDelta::Seconds(5)));
Markus Handell28c71802021-11-08 10:11:55 +0100454 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200455 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200456
Henrik Boström91aa7322020-04-28 12:24:33 +0200457 void TriggerCpuUnderuse() {
458 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200459 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200460 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200461 event.Set();
462 });
Markus Handell2cfc1af2022-08-19 08:16:48 +0000463 ASSERT_TRUE(event.Wait(TimeDelta::Seconds(5)));
Markus Handell28c71802021-11-08 10:11:55 +0100464 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200465 }
kthelgason876222f2016-11-29 01:44:11 -0800466
Henrik Boström91aa7322020-04-28 12:24:33 +0200467 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200468 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200469 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200470 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200471 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200472 event.Set();
473 });
Markus Handell2cfc1af2022-08-19 08:16:48 +0000474 ASSERT_TRUE(event.Wait(TimeDelta::Seconds(5)));
Markus Handell28c71802021-11-08 10:11:55 +0100475 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200476 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200477 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200478 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200479 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200480 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200481 event.Set();
482 });
Markus Handell2cfc1af2022-08-19 08:16:48 +0000483 ASSERT_TRUE(event.Wait(TimeDelta::Seconds(5)));
Markus Handell28c71802021-11-08 10:11:55 +0100484 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200485 }
486
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200487 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100488 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200489 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
490 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200491 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700492};
493
Noah Richards51db4212019-06-12 06:59:12 -0700494// Simulates simulcast behavior and makes highest stream resolutions divisible
495// by 4.
496class CroppingVideoStreamFactory
497 : public VideoEncoderConfig::VideoStreamFactoryInterface {
498 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200499 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700500
501 private:
502 std::vector<VideoStream> CreateEncoderStreams(
Jonas Oreland80c87d72022-09-29 15:01:09 +0200503 int frame_width,
504 int frame_height,
Noah Richards51db4212019-06-12 06:59:12 -0700505 const VideoEncoderConfig& encoder_config) override {
506 std::vector<VideoStream> streams = test::CreateVideoStreams(
Jonas Oreland80c87d72022-09-29 15:01:09 +0200507 frame_width - frame_width % 4, frame_height - frame_height % 4,
508 encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700509 return streams;
510 }
Noah Richards51db4212019-06-12 06:59:12 -0700511};
512
sprangb1ca0732017-02-01 08:38:12 -0800513class AdaptingFrameForwarder : public test::FrameForwarder {
514 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200515 explicit AdaptingFrameForwarder(TimeController* time_controller)
516 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700517 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800518
519 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200520 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800521 adaptation_enabled_ = enabled;
522 }
523
asaperssonfab67072017-04-04 05:51:49 -0700524 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200525 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800526 return adaptation_enabled_;
527 }
528
Henrik Boström1124ed12021-02-25 10:30:39 +0100529 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
530 // the resolution or frame rate was different than it is currently. If
531 // something else is modified, such as encoder resolutions, but the resolution
532 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700533 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200534 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700535 return last_wants_;
536 }
537
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200538 absl::optional<int> last_sent_width() const { return last_width_; }
539 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800540
sprangb1ca0732017-02-01 08:38:12 -0800541 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200542 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +0100543 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200544
sprangb1ca0732017-02-01 08:38:12 -0800545 int cropped_width = 0;
546 int cropped_height = 0;
547 int out_width = 0;
548 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700549 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000550 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
551 << "w=" << video_frame.width()
552 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700553 if (adapter_.AdaptFrameResolution(
554 video_frame.width(), video_frame.height(),
555 video_frame.timestamp_us() * 1000, &cropped_width,
556 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100557 VideoFrame adapted_frame =
558 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200559 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100560 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200561 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100562 .set_timestamp_ms(99)
563 .set_rotation(kVideoRotation_0)
564 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100565 if (video_frame.has_update_rect()) {
566 adapted_frame.set_update_rect(
567 video_frame.update_rect().ScaleWithFrame(
568 video_frame.width(), video_frame.height(), 0, 0,
569 video_frame.width(), video_frame.height(), out_width,
570 out_height));
571 }
sprangc5d62e22017-04-02 23:53:04 -0700572 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800573 last_width_.emplace(adapted_frame.width());
574 last_height_.emplace(adapted_frame.height());
575 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200576 last_width_ = absl::nullopt;
577 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700578 }
sprangb1ca0732017-02-01 08:38:12 -0800579 } else {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000580 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800581 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800582 last_width_.emplace(video_frame.width());
583 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800584 }
585 }
586
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200587 void OnOutputFormatRequest(int width, int height) {
588 absl::optional<std::pair<int, int>> target_aspect_ratio =
589 std::make_pair(width, height);
590 absl::optional<int> max_pixel_count = width * height;
591 absl::optional<int> max_fps;
592 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
593 max_fps);
594 }
595
sprangb1ca0732017-02-01 08:38:12 -0800596 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
597 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200598 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100599 rtc::VideoSinkWants prev_wants = sink_wants_locked();
600 bool did_adapt =
601 prev_wants.max_pixel_count != wants.max_pixel_count ||
602 prev_wants.target_pixel_count != wants.target_pixel_count ||
603 prev_wants.max_framerate_fps != wants.max_framerate_fps;
604 if (did_adapt) {
605 last_wants_ = prev_wants;
606 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100607 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200608 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800609 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200610
Erik Språng5e13d052022-08-02 11:42:49 +0200611 void RequestRefreshFrame() override { ++refresh_frames_requested_; }
612
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200613 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800614 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200615 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
616 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200617 absl::optional<int> last_width_;
618 absl::optional<int> last_height_;
Erik Språng5e13d052022-08-02 11:42:49 +0200619 int refresh_frames_requested_{0};
sprangb1ca0732017-02-01 08:38:12 -0800620};
sprangc5d62e22017-04-02 23:53:04 -0700621
Niels Möller213618e2018-07-24 09:29:58 +0200622// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700623class MockableSendStatisticsProxy : public SendStatisticsProxy {
624 public:
625 MockableSendStatisticsProxy(Clock* clock,
626 const VideoSendStream::Config& config,
Jonas Oreland8ca06132022-03-14 12:52:48 +0100627 VideoEncoderConfig::ContentType content_type,
Jonas Orelande62c2f22022-03-29 11:04:48 +0200628 const FieldTrialsView& field_trials)
Jonas Oreland8ca06132022-03-14 12:52:48 +0100629 : SendStatisticsProxy(clock, config, content_type, field_trials) {}
sprangc5d62e22017-04-02 23:53:04 -0700630
631 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200632 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700633 if (mock_stats_)
634 return *mock_stats_;
635 return SendStatisticsProxy::GetStats();
636 }
637
Niels Möller213618e2018-07-24 09:29:58 +0200638 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200639 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200640 if (mock_stats_)
641 return mock_stats_->input_frame_rate;
642 return SendStatisticsProxy::GetInputFrameRate();
643 }
sprangc5d62e22017-04-02 23:53:04 -0700644 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200645 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700646 mock_stats_.emplace(stats);
647 }
648
649 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200650 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700651 mock_stats_.reset();
652 }
653
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200654 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
655 on_frame_dropped_ = std::move(callback);
656 }
657
sprangc5d62e22017-04-02 23:53:04 -0700658 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200659 void OnFrameDropped(DropReason reason) override {
660 SendStatisticsProxy::OnFrameDropped(reason);
661 if (on_frame_dropped_)
662 on_frame_dropped_(reason);
663 }
664
Markus Handella3765182020-07-08 13:13:32 +0200665 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200666 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200667 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700668};
669
Markus Handellb4e96d42021-11-05 12:00:55 +0100670class SimpleVideoStreamEncoderFactory {
671 public:
672 class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
673 public:
674 using VideoStreamEncoder::VideoStreamEncoder;
675 ~AdaptedVideoStreamEncoder() { Stop(); }
676 };
677
Markus Handell8d87c462021-12-16 11:37:16 +0100678 class MockFakeEncoder : public test::FakeEncoder {
679 public:
680 using FakeEncoder::FakeEncoder;
681 MOCK_METHOD(CodecSpecificInfo,
682 EncodeHook,
683 (EncodedImage & encoded_image,
684 rtc::scoped_refptr<EncodedImageBuffer> buffer),
685 (override));
686 };
687
Markus Handellee225432021-11-29 12:35:12 +0100688 SimpleVideoStreamEncoderFactory() {
Markus Handellb4e96d42021-11-05 12:00:55 +0100689 encoder_settings_.encoder_factory = &encoder_factory_;
Markus Handellee225432021-11-29 12:35:12 +0100690 encoder_settings_.bitrate_allocator_factory =
691 bitrate_allocator_factory_.get();
Markus Handellb4e96d42021-11-05 12:00:55 +0100692 }
693
Markus Handell818e7fb2021-12-30 13:01:33 +0100694 std::unique_ptr<AdaptedVideoStreamEncoder> CreateWithEncoderQueue(
Markus Handellee225432021-11-29 12:35:12 +0100695 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100696 std::unique_ptr<TaskQueueBase, TaskQueueDeleter> encoder_queue,
Jonas Orelande62c2f22022-03-29 11:04:48 +0200697 const FieldTrialsView* field_trials = nullptr) {
Markus Handellb4e96d42021-11-05 12:00:55 +0100698 auto result = std::make_unique<AdaptedVideoStreamEncoder>(
699 time_controller_.GetClock(),
700 /*number_of_cores=*/1,
701 /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
Markus Handell8e4197b2022-05-30 15:45:28 +0200702 std::make_unique<CpuOveruseDetectorProxy>(
703 /*stats_proxy=*/nullptr,
704 field_trials ? *field_trials : field_trials_),
Markus Handellee225432021-11-29 12:35:12 +0100705 std::move(zero_hertz_adapter), std::move(encoder_queue),
Markus Handellb4e96d42021-11-05 12:00:55 +0100706 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100707 kVideoBitrateAllocation,
708 field_trials ? *field_trials : field_trials_);
Markus Handellb4e96d42021-11-05 12:00:55 +0100709 result->SetSink(&sink_, /*rotation_applied=*/false);
710 return result;
711 }
712
Markus Handell818e7fb2021-12-30 13:01:33 +0100713 std::unique_ptr<AdaptedVideoStreamEncoder> Create(
714 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
715 TaskQueueBase** encoder_queue_ptr = nullptr) {
716 auto encoder_queue =
717 time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
718 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
719 if (encoder_queue_ptr)
720 *encoder_queue_ptr = encoder_queue.get();
721 return CreateWithEncoderQueue(std::move(zero_hertz_adapter),
722 std::move(encoder_queue));
723 }
724
Markus Handell9a478b52021-11-18 16:07:01 +0100725 void DepleteTaskQueues() { time_controller_.AdvanceTime(TimeDelta::Zero()); }
Markus Handell8d87c462021-12-16 11:37:16 +0100726 MockFakeEncoder& GetMockFakeEncoder() { return mock_fake_encoder_; }
Markus Handell9a478b52021-11-18 16:07:01 +0100727
Markus Handell818e7fb2021-12-30 13:01:33 +0100728 GlobalSimulatedTimeController* GetTimeController() {
729 return &time_controller_;
730 }
731
Markus Handellb4e96d42021-11-05 12:00:55 +0100732 private:
733 class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
734 public:
735 ~NullEncoderSink() override = default;
736 void OnEncoderConfigurationChanged(
737 std::vector<VideoStream> streams,
738 bool is_svc,
739 VideoEncoderConfig::ContentType content_type,
740 int min_transmit_bitrate_bps) override {}
741 void OnBitrateAllocationUpdated(
742 const VideoBitrateAllocation& allocation) override {}
743 void OnVideoLayersAllocationUpdated(
744 VideoLayersAllocation allocation) override {}
745 Result OnEncodedImage(
746 const EncodedImage& encoded_image,
747 const CodecSpecificInfo* codec_specific_info) override {
748 return Result(EncodedImageCallback::Result::OK);
749 }
750 };
751
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100752 test::ScopedKeyValueConfig field_trials_;
Philipp Hanckea204ad22022-07-08 18:43:25 +0200753 GlobalSimulatedTimeController time_controller_{Timestamp::Zero()};
Markus Handellee225432021-11-29 12:35:12 +0100754 std::unique_ptr<TaskQueueFactory> task_queue_factory_{
755 time_controller_.CreateTaskQueueFactory()};
756 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_ =
757 std::make_unique<MockableSendStatisticsProxy>(
758 time_controller_.GetClock(),
759 VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 12:52:48 +0100760 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
761 field_trials_);
Markus Handellee225432021-11-29 12:35:12 +0100762 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_ =
763 CreateBuiltinVideoBitrateAllocatorFactory();
764 VideoStreamEncoderSettings encoder_settings_{
765 VideoEncoder::Capabilities(/*loss_notification=*/false)};
Markus Handell8d87c462021-12-16 11:37:16 +0100766 MockFakeEncoder mock_fake_encoder_{time_controller_.GetClock()};
767 test::VideoEncoderProxyFactory encoder_factory_{&mock_fake_encoder_};
Markus Handellb4e96d42021-11-05 12:00:55 +0100768 NullEncoderSink sink_;
769};
770
771class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
772 public:
773 MOCK_METHOD(void, Initialize, (Callback * callback), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100774 MOCK_METHOD(void,
775 SetZeroHertzModeEnabled,
776 (absl::optional<ZeroHertzModeParams>),
777 (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100778 MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
Markus Handellee225432021-11-29 12:35:12 +0100779 MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override));
780 MOCK_METHOD(void, UpdateFrameRate, (), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100781 MOCK_METHOD(void,
782 UpdateLayerQualityConvergence,
Markus Handell5a77e512022-09-01 12:51:50 +0000783 (size_t spatial_index, bool converged),
Markus Handell8d87c462021-12-16 11:37:16 +0100784 (override));
785 MOCK_METHOD(void,
786 UpdateLayerStatus,
Markus Handell5a77e512022-09-01 12:51:50 +0000787 (size_t spatial_index, bool enabled),
Markus Handell8d87c462021-12-16 11:37:16 +0100788 (override));
Markus Handell818e7fb2021-12-30 13:01:33 +0100789 MOCK_METHOD(void, ProcessKeyFrameRequest, (), (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100790};
791
philipel9b058032020-02-10 11:30:00 +0100792class MockEncoderSelector
793 : public VideoEncoderFactory::EncoderSelectorInterface {
794 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200795 MOCK_METHOD(void,
796 OnCurrentEncoder,
797 (const SdpVideoFormat& format),
798 (override));
799 MOCK_METHOD(absl::optional<SdpVideoFormat>,
800 OnAvailableBitrate,
801 (const DataRate& rate),
802 (override));
philipel6daa3042022-04-11 10:48:28 +0200803 MOCK_METHOD(absl::optional<SdpVideoFormat>,
804 OnResolutionChange,
805 (const RenderResolution& resolution),
806 (override));
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200807 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100808};
809
Markus Handell2e0f4f02021-12-21 19:14:58 +0100810class MockVideoSourceInterface : public rtc::VideoSourceInterface<VideoFrame> {
811 public:
812 MOCK_METHOD(void,
813 AddOrUpdateSink,
814 (rtc::VideoSinkInterface<VideoFrame>*,
815 const rtc::VideoSinkWants&),
816 (override));
817 MOCK_METHOD(void,
818 RemoveSink,
819 (rtc::VideoSinkInterface<VideoFrame>*),
820 (override));
821 MOCK_METHOD(void, RequestRefreshFrame, (), (override));
822};
823
perkj803d97f2016-11-01 11:45:46 -0700824} // namespace
825
mflodmancc3d4422017-08-03 08:27:51 -0700826class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700827 public:
Markus Handell2cfc1af2022-08-19 08:16:48 +0000828 static constexpr TimeDelta kDefaultTimeout = TimeDelta::Seconds(1);
perkj26091b12016-09-01 01:17:40 -0700829
mflodmancc3d4422017-08-03 08:27:51 -0700830 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700831 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700832 codec_width_(320),
833 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200834 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200835 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200836 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700837 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200838 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700839 video_send_config_,
Jonas Oreland8ca06132022-03-14 12:52:48 +0100840 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
841 field_trials_)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200842 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700843
844 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700845 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700846 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200847 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800848 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200849 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200850 video_send_config_.rtp.payload_name = "FAKE";
851 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700852
Per512ecb32016-09-23 15:52:06 +0200853 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200854 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200855 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
856 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
857 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100858 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700859
Niels Möllerf1338562018-04-26 09:51:47 +0200860 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800861 }
862
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100863 void ConfigureEncoder(
864 VideoEncoderConfig video_encoder_config,
865 VideoStreamEncoder::BitrateAllocationCallbackType
866 allocation_callback_type =
867 VideoStreamEncoder::BitrateAllocationCallbackType::
Erik Språnge4589cb2022-04-06 16:44:30 +0200868 kVideoBitrateAllocationWhenScreenSharing,
869 int num_cores = 1) {
mflodmancc3d4422017-08-03 08:27:51 -0700870 if (video_stream_encoder_)
871 video_stream_encoder_->Stop();
Markus Handell9a478b52021-11-18 16:07:01 +0100872
873 auto encoder_queue = GetTaskQueueFactory()->CreateTaskQueue(
874 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
875 TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
876 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter =
877 FrameCadenceAdapterInterface::Create(time_controller_.GetClock(),
Jonas Oreland8ca06132022-03-14 12:52:48 +0100878 encoder_queue_ptr, field_trials_);
Markus Handell9a478b52021-11-18 16:07:01 +0100879 video_stream_encoder_ = std::make_unique<VideoStreamEncoderUnderTest>(
880 &time_controller_, std::move(cadence_adapter), std::move(encoder_queue),
881 stats_proxy_.get(), video_send_config_.encoder_settings,
Erik Språnge4589cb2022-04-06 16:44:30 +0200882 allocation_callback_type, field_trials_, num_cores);
Asa Persson606d3cb2021-10-04 10:07:11 +0200883 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 08:27:51 -0700884 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700885 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 10:07:11 +0200886 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 08:27:51 -0700887 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Florent Castelliacabb362022-10-18 17:05:16 +0200888 kMaxPayloadLength, nullptr);
mflodmancc3d4422017-08-03 08:27:51 -0700889 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800890 }
891
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100892 void ResetEncoder(const std::string& payload_name,
893 size_t num_streams,
894 size_t num_temporal_layers,
895 unsigned char num_spatial_layers,
896 bool screenshare,
897 VideoStreamEncoder::BitrateAllocationCallbackType
898 allocation_callback_type =
899 VideoStreamEncoder::BitrateAllocationCallbackType::
Erik Språnge4589cb2022-04-06 16:44:30 +0200900 kVideoBitrateAllocationWhenScreenSharing,
901 int num_cores = 1) {
Niels Möller259a4972018-04-05 15:36:51 +0200902 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800903
904 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200905 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
906 num_streams, &video_encoder_config);
907 for (auto& layer : video_encoder_config.simulcast_layers) {
908 layer.num_temporal_layers = num_temporal_layers;
909 layer.max_framerate = kDefaultFramerate;
910 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100911 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 10:07:11 +0200912 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 07:06:52 -0700913 video_encoder_config.content_type =
914 screenshare ? VideoEncoderConfig::ContentType::kScreen
915 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700916 if (payload_name == "VP9") {
917 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
918 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200919 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700920 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200921 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
922 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700923 }
Erik Språnge4589cb2022-04-06 16:44:30 +0200924 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type,
925 num_cores);
perkj26091b12016-09-01 01:17:40 -0700926 }
927
sprang57c2fff2017-01-16 06:24:02 -0800928 VideoFrame CreateFrame(int64_t ntp_time_ms,
929 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200930 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200931 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200932 destruction_event, codec_width_, codec_height_))
933 .set_ntp_time_ms(ntp_time_ms)
934 .set_timestamp_ms(99)
935 .set_rotation(kVideoRotation_0)
936 .build();
perkj26091b12016-09-01 01:17:40 -0700937 }
938
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100939 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
940 rtc::Event* destruction_event,
941 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200942 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200943 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200944 destruction_event, codec_width_, codec_height_))
945 .set_ntp_time_ms(ntp_time_ms)
946 .set_timestamp_ms(99)
947 .set_rotation(kVideoRotation_0)
948 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
949 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100950 }
951
sprang57c2fff2017-01-16 06:24:02 -0800952 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200953 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
954 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200955 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200956 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200957 .set_ntp_time_ms(ntp_time_ms)
958 .set_timestamp_ms(ntp_time_ms)
959 .set_rotation(kVideoRotation_0)
960 .build();
perkj803d97f2016-11-01 11:45:46 -0700961 }
962
Evan Shrubsole895556e2020-10-05 09:15:13 +0200963 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200964 return VideoFrame::Builder()
965 .set_video_frame_buffer(NV12Buffer::Create(width, height))
966 .set_ntp_time_ms(ntp_time_ms)
967 .set_timestamp_ms(ntp_time_ms)
968 .set_rotation(kVideoRotation_0)
969 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200970 }
971
Noah Richards51db4212019-06-12 06:59:12 -0700972 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
973 rtc::Event* destruction_event,
974 int width,
975 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200976 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200977 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200978 destruction_event, width, height))
979 .set_ntp_time_ms(ntp_time_ms)
980 .set_timestamp_ms(99)
981 .set_rotation(kVideoRotation_0)
982 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700983 }
984
Evan Shrubsole895556e2020-10-05 09:15:13 +0200985 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
986 rtc::Event* destruction_event,
987 int width,
988 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200989 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200990 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200991 destruction_event, width, height))
992 .set_ntp_time_ms(ntp_time_ms)
993 .set_timestamp_ms(99)
994 .set_rotation(kVideoRotation_0)
995 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200996 }
997
Noah Richards51db4212019-06-12 06:59:12 -0700998 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
999 rtc::Event* destruction_event) const {
1000 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
1001 codec_height_);
1002 }
1003
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001004 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001005 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001006 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001007
1008 video_source_.IncomingCapturedFrame(
1009 CreateFrame(1, codec_width_, codec_height_));
1010 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +02001011 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001012 }
1013
sprang4847ae62017-06-27 07:06:52 -07001014 void WaitForEncodedFrame(int64_t expected_ntp_time) {
1015 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001016 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001017 }
1018
Markus Handell2cfc1af2022-08-19 08:16:48 +00001019 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, TimeDelta timeout) {
1020 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001021 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001022 return ok;
1023 }
1024
1025 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
1026 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001027 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001028 }
1029
1030 void ExpectDroppedFrame() {
1031 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001032 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001033 }
1034
Markus Handell2cfc1af2022-08-19 08:16:48 +00001035 bool WaitForFrame(TimeDelta timeout) {
1036 bool ok = sink_.WaitForFrame(timeout);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001037 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001038 return ok;
1039 }
1040
perkj26091b12016-09-01 01:17:40 -07001041 class TestEncoder : public test::FakeEncoder {
1042 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001043 explicit TestEncoder(TimeController* time_controller)
1044 : FakeEncoder(time_controller->GetClock()),
1045 time_controller_(time_controller) {
1046 RTC_DCHECK(time_controller_);
1047 }
perkj26091b12016-09-01 01:17:40 -07001048
Erik Språngaed30702018-11-05 12:57:17 +01001049 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +02001050 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +02001051 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +01001052 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001053 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +01001054 info.scaling_settings = VideoEncoder::ScalingSettings(
1055 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001056 }
1057 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001058 for (int i = 0; i < kMaxSpatialLayers; ++i) {
1059 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001060 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001061 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001062 for (int tid = 0; tid < num_layers; ++tid)
1063 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001064 }
1065 }
Erik Språngaed30702018-11-05 12:57:17 +01001066 }
Sergey Silkin6456e352019-07-08 17:56:40 +02001067
1068 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001069 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001070 info.apply_alignment_to_all_simulcast_layers =
1071 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001072 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001073 if (is_qp_trusted_.has_value()) {
1074 info.is_qp_trusted = is_qp_trusted_;
1075 }
Erik Språngaed30702018-11-05 12:57:17 +01001076 return info;
kthelgason876222f2016-11-29 01:44:11 -08001077 }
1078
Erik Språngb7cb7b52019-02-26 15:52:33 +01001079 int32_t RegisterEncodeCompleteCallback(
1080 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +02001081 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001082 encoded_image_callback_ = callback;
1083 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
1084 }
1085
perkjfa10b552016-10-02 23:45:26 -07001086 void ContinueEncode() { continue_encode_event_.Set(); }
1087
1088 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
1089 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +02001090 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -07001091 EXPECT_EQ(timestamp_, timestamp);
1092 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
1093 }
1094
kthelgason2fc52542017-03-03 00:24:41 -08001095 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +02001096 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -08001097 quality_scaling_ = b;
1098 }
kthelgasonad9010c2017-02-14 00:46:51 -08001099
Chunbo Hua46ad2512022-12-06 23:56:33 +08001100 void SetRequestedResolutionAlignment(
1101 uint32_t requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +02001102 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001103 requested_resolution_alignment_ = requested_resolution_alignment;
1104 }
1105
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001106 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
1107 MutexLock lock(&local_mutex_);
1108 apply_alignment_to_all_simulcast_layers_ = b;
1109 }
1110
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001111 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +02001112 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001113 is_hardware_accelerated_ = is_hardware_accelerated;
1114 }
1115
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001116 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
1117 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +02001118 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001119 temporal_layers_supported_[spatial_idx] = supported;
1120 }
1121
Sergey Silkin6456e352019-07-08 17:56:40 +02001122 void SetResolutionBitrateLimits(
1123 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +02001124 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +02001125 resolution_bitrate_limits_ = thresholds;
1126 }
1127
sprangfe627f32017-03-29 08:24:59 -07001128 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +02001129 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -07001130 force_init_encode_failed_ = force_failure;
1131 }
1132
Niels Möller6bb5ab92019-01-11 11:11:10 +01001133 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +02001134 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001135 rate_factor_ = rate_factor;
1136 }
1137
Erik Språngd7329ca2019-02-21 21:19:53 +01001138 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +02001139 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001140 return last_framerate_;
1141 }
1142
Erik Språngd7329ca2019-02-21 21:19:53 +01001143 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +02001144 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001145 return last_update_rect_;
1146 }
1147
Niels Möller87e2d782019-03-07 10:18:23 +01001148 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +02001149 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001150 return last_frame_types_;
1151 }
1152
1153 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +01001154 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +01001155 keyframe ? VideoFrameType::kVideoFrameKey
1156 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01001157 {
Markus Handella3765182020-07-08 13:13:32 +02001158 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001159 last_frame_types_ = frame_type;
1160 }
Niels Möllerb859b322019-03-07 12:40:01 +01001161 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +01001162 }
1163
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001164 void InjectEncodedImage(const EncodedImage& image,
1165 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +02001166 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001167 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001168 }
1169
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001170 void SetEncodedImageData(
1171 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +02001172 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001173 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001174 }
1175
Erik Språngd7329ca2019-02-21 21:19:53 +01001176 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +02001177 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001178 expect_null_frame_ = true;
1179 }
1180
Erik Språng5056af02019-09-02 15:53:11 +02001181 absl::optional<VideoEncoder::RateControlParameters>
1182 GetAndResetLastRateControlSettings() {
1183 auto settings = last_rate_control_settings_;
1184 last_rate_control_settings_.reset();
1185 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +01001186 }
1187
Henrik Boström56db9ff2021-03-24 09:06:45 +01001188 int GetLastInputWidth() const {
1189 MutexLock lock(&local_mutex_);
1190 return last_input_width_;
1191 }
1192
1193 int GetLastInputHeight() const {
1194 MutexLock lock(&local_mutex_);
1195 return last_input_height_;
1196 }
1197
Evan Shrubsole895556e2020-10-05 09:15:13 +02001198 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1199 MutexLock lock(&local_mutex_);
1200 return last_input_pixel_format_;
1201 }
1202
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001203 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001204 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001205 return num_set_rates_;
1206 }
1207
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001208 void SetPreferredPixelFormats(
1209 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1210 pixel_formats) {
1211 MutexLock lock(&local_mutex_);
1212 preferred_pixel_formats_ = std::move(pixel_formats);
1213 }
1214
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001215 void SetIsQpTrusted(absl::optional<bool> trusted) {
1216 MutexLock lock(&local_mutex_);
1217 is_qp_trusted_ = trusted;
1218 }
1219
Erik Språnge4589cb2022-04-06 16:44:30 +02001220 VideoCodecComplexity LastEncoderComplexity() {
1221 MutexLock lock(&local_mutex_);
1222 return last_encoder_complexity_;
1223 }
1224
perkjfa10b552016-10-02 23:45:26 -07001225 private:
perkj26091b12016-09-01 01:17:40 -07001226 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001227 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001228 {
Markus Handella3765182020-07-08 13:13:32 +02001229 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001230 if (expect_null_frame_) {
1231 EXPECT_EQ(input_image.timestamp(), 0u);
1232 EXPECT_EQ(input_image.width(), 1);
1233 last_frame_types_ = *frame_types;
1234 expect_null_frame_ = false;
1235 } else {
1236 EXPECT_GT(input_image.timestamp(), timestamp_);
1237 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1238 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1239 }
perkj26091b12016-09-01 01:17:40 -07001240
1241 timestamp_ = input_image.timestamp();
1242 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001243 last_input_width_ = input_image.width();
1244 last_input_height_ = input_image.height();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001245 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001246 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001247 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001248 }
Niels Möllerb859b322019-03-07 12:40:01 +01001249 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001250 return result;
1251 }
1252
Niels Möller08ae7ce2020-09-23 15:58:12 +02001253 CodecSpecificInfo EncodeHook(
1254 EncodedImage& encoded_image,
1255 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001256 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001257 {
1258 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001259 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001260 }
1261 MutexLock lock(&local_mutex_);
1262 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001263 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001264 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001265 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001266 }
1267
sprangfe627f32017-03-29 08:24:59 -07001268 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001269 const Settings& settings) override {
1270 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001271
Markus Handella3765182020-07-08 13:13:32 +02001272 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001273 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001274
Erik Språng82fad3d2018-03-21 09:57:23 +01001275 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001276 // Simulate setting up temporal layers, in order to validate the life
1277 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001278 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001279 frame_buffer_controller_ =
1280 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001281 }
Erik Språnge4589cb2022-04-06 16:44:30 +02001282
1283 last_encoder_complexity_ = config->GetVideoEncoderComplexity();
1284
Erik Språngb7cb7b52019-02-26 15:52:33 +01001285 if (force_init_encode_failed_) {
1286 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001287 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001288 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001289
Erik Språngb7cb7b52019-02-26 15:52:33 +01001290 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001291 return res;
1292 }
1293
Erik Språngb7cb7b52019-02-26 15:52:33 +01001294 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001295 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001296 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1297 initialized_ = EncoderState::kUninitialized;
1298 return FakeEncoder::Release();
1299 }
1300
Erik Språng16cb8f52019-04-12 13:59:09 +02001301 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001302 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001303 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001304 VideoBitrateAllocation adjusted_rate_allocation;
1305 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1306 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001307 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001308 adjusted_rate_allocation.SetBitrate(
1309 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001310 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001311 rate_factor_));
1312 }
1313 }
1314 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001315 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001316 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001317 RateControlParameters adjusted_paramters = parameters;
1318 adjusted_paramters.bitrate = adjusted_rate_allocation;
1319 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001320 }
1321
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001322 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001323 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001324 enum class EncoderState {
1325 kUninitialized,
1326 kInitializationFailed,
1327 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001328 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
perkj26091b12016-09-01 01:17:40 -07001329 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001330 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1331 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1332 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1333 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1334 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
Chunbo Hua46ad2512022-12-06 23:56:33 +08001335 uint32_t requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001336 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1337 false;
Markus Handella3765182020-07-08 13:13:32 +02001338 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001339 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1340 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001341 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001342 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001343 absl::optional<bool>
1344 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001345 local_mutex_);
1346 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1347 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1348 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001349 absl::optional<VideoEncoder::RateControlParameters>
1350 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001351 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1352 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001353 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001354 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001355 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1356 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001357 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 17:56:40 +02001358 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001359 RTC_GUARDED_BY(local_mutex_);
1360 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001361 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1362 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001363 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1364 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001365 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
Erik Språnge4589cb2022-04-06 16:44:30 +02001366 VideoCodecComplexity last_encoder_complexity_ RTC_GUARDED_BY(local_mutex_){
1367 VideoCodecComplexity::kComplexityNormal};
perkj26091b12016-09-01 01:17:40 -07001368 };
1369
mflodmancc3d4422017-08-03 08:27:51 -07001370 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001371 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001372 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1373 : time_controller_(time_controller), test_encoder_(test_encoder) {
1374 RTC_DCHECK(time_controller_);
1375 }
perkj26091b12016-09-01 01:17:40 -07001376
perkj26091b12016-09-01 01:17:40 -07001377 void WaitForEncodedFrame(int64_t expected_ntp_time) {
Markus Handell2cfc1af2022-08-19 08:16:48 +00001378 EXPECT_TRUE(TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeout));
sprang4847ae62017-06-27 07:06:52 -07001379 }
1380
1381 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
Markus Handell2cfc1af2022-08-19 08:16:48 +00001382 TimeDelta timeout) {
perkj26091b12016-09-01 01:17:40 -07001383 uint32_t timestamp = 0;
Markus Handell2cfc1af2022-08-19 08:16:48 +00001384 if (!WaitForFrame(timeout))
sprang4847ae62017-06-27 07:06:52 -07001385 return false;
perkj26091b12016-09-01 01:17:40 -07001386 {
Markus Handella3765182020-07-08 13:13:32 +02001387 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001388 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001389 }
1390 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001391 return true;
perkj26091b12016-09-01 01:17:40 -07001392 }
1393
sprangb1ca0732017-02-01 08:38:12 -08001394 void WaitForEncodedFrame(uint32_t expected_width,
1395 uint32_t expected_height) {
Markus Handell2cfc1af2022-08-19 08:16:48 +00001396 EXPECT_TRUE(WaitForFrame(kDefaultTimeout));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001397 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001398 }
1399
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001400 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001401 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001402 uint32_t width = 0;
1403 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001404 {
Markus Handella3765182020-07-08 13:13:32 +02001405 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001406 width = last_width_;
1407 height = last_height_;
1408 }
1409 EXPECT_EQ(expected_height, height);
1410 EXPECT_EQ(expected_width, width);
1411 }
1412
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001413 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1414 VideoRotation rotation;
1415 {
Markus Handella3765182020-07-08 13:13:32 +02001416 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001417 rotation = last_rotation_;
1418 }
1419 EXPECT_EQ(expected_rotation, rotation);
1420 }
1421
Markus Handell2cfc1af2022-08-19 08:16:48 +00001422 void ExpectDroppedFrame() {
1423 EXPECT_FALSE(WaitForFrame(TimeDelta::Millis(100)));
1424 }
kthelgason2bc68642017-02-07 07:02:22 -08001425
Markus Handell2cfc1af2022-08-19 08:16:48 +00001426 bool WaitForFrame(TimeDelta timeout) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001427 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +01001428 time_controller_->AdvanceTime(TimeDelta::Zero());
Markus Handell2cfc1af2022-08-19 08:16:48 +00001429 bool ret = encoded_frame_event_.Wait(timeout);
Markus Handell28c71802021-11-08 10:11:55 +01001430 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001431 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001432 }
1433
perkj26091b12016-09-01 01:17:40 -07001434 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001435 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001436 expect_frames_ = false;
1437 }
1438
asaperssonfab67072017-04-04 05:51:49 -07001439 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001440 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001441 return number_of_reconfigurations_;
1442 }
1443
asaperssonfab67072017-04-04 05:51:49 -07001444 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001445 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001446 return min_transmit_bitrate_bps_;
1447 }
1448
Erik Språngd7329ca2019-02-21 21:19:53 +01001449 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001450 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001451 num_expected_layers_ = num_layers;
1452 }
1453
Erik Språngb7cb7b52019-02-26 15:52:33 +01001454 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001455 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001456 return last_capture_time_ms_;
1457 }
1458
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001459 const EncodedImage& GetLastEncodedImage() {
1460 MutexLock lock(&mutex_);
1461 return last_encoded_image_;
1462 }
1463
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001464 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001465 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001466 return std::move(last_encoded_image_data_);
1467 }
1468
Per Kjellanderdcef6412020-10-07 15:09:05 +02001469 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1470 MutexLock lock(&mutex_);
1471 return last_bitrate_allocation_;
1472 }
1473
1474 int number_of_bitrate_allocations() const {
1475 MutexLock lock(&mutex_);
1476 return number_of_bitrate_allocations_;
1477 }
1478
Per Kjellandera9434842020-10-15 17:53:22 +02001479 VideoLayersAllocation GetLastVideoLayersAllocation() {
1480 MutexLock lock(&mutex_);
1481 return last_layers_allocation_;
1482 }
1483
1484 int number_of_layers_allocations() const {
1485 MutexLock lock(&mutex_);
1486 return number_of_layers_allocations_;
1487 }
1488
perkj26091b12016-09-01 01:17:40 -07001489 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001490 Result OnEncodedImage(
1491 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001492 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001493 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001494 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001495 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001496 last_encoded_image_data_ = std::vector<uint8_t>(
1497 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001498 uint32_t timestamp = encoded_image.Timestamp();
1499 if (last_timestamp_ != timestamp) {
1500 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001501 last_width_ = encoded_image._encodedWidth;
1502 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001503 } else {
1504 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001505 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1506 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001507 }
1508 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001509 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001510 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001511 if (num_received_layers_ == num_expected_layers_) {
1512 encoded_frame_event_.Set();
1513 }
sprangb1ca0732017-02-01 08:38:12 -08001514 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001515 }
1516
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001517 void OnEncoderConfigurationChanged(
1518 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001519 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001520 VideoEncoderConfig::ContentType content_type,
1521 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001522 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001523 ++number_of_reconfigurations_;
1524 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1525 }
1526
Per Kjellanderdcef6412020-10-07 15:09:05 +02001527 void OnBitrateAllocationUpdated(
1528 const VideoBitrateAllocation& allocation) override {
1529 MutexLock lock(&mutex_);
1530 ++number_of_bitrate_allocations_;
1531 last_bitrate_allocation_ = allocation;
1532 }
1533
Per Kjellandera9434842020-10-15 17:53:22 +02001534 void OnVideoLayersAllocationUpdated(
1535 VideoLayersAllocation allocation) override {
1536 MutexLock lock(&mutex_);
1537 ++number_of_layers_allocations_;
1538 last_layers_allocation_ = allocation;
1539 rtc::StringBuilder log;
1540 for (const auto& layer : allocation.active_spatial_layers) {
1541 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1542 << "[";
1543 for (const auto target_bitrate :
1544 layer.target_bitrate_per_temporal_layer) {
1545 log << target_bitrate.kbps() << ",";
1546 }
1547 log << "]";
1548 }
Harald Alvestrand97597c02021-11-04 12:01:23 +00001549 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 17:53:22 +02001550 }
1551
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001552 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001553 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001554 TestEncoder* test_encoder_;
1555 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001556 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001557 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001558 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001559 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001560 uint32_t last_height_ = 0;
1561 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001562 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001563 size_t num_expected_layers_ = 1;
1564 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001565 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001566 int number_of_reconfigurations_ = 0;
1567 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001568 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1569 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001570 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1571 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001572 };
1573
Sergey Silkin5ee69672019-07-02 14:18:34 +02001574 class VideoBitrateAllocatorProxyFactory
1575 : public VideoBitrateAllocatorFactory {
1576 public:
1577 VideoBitrateAllocatorProxyFactory()
1578 : bitrate_allocator_factory_(
1579 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1580
1581 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1582 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001583 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001584 codec_config_ = codec;
1585 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1586 }
1587
1588 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001589 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001590 return codec_config_;
1591 }
1592
1593 private:
1594 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1595
Markus Handella3765182020-07-08 13:13:32 +02001596 mutable Mutex mutex_;
1597 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001598 };
1599
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001600 Clock* clock() { return time_controller_.GetClock(); }
1601 void AdvanceTime(TimeDelta duration) {
1602 time_controller_.AdvanceTime(duration);
1603 }
1604
1605 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1606
1607 protected:
1608 virtual TaskQueueFactory* GetTaskQueueFactory() {
1609 return time_controller_.GetTaskQueueFactory();
1610 }
1611
Jonas Orelandc7f691a2022-03-09 15:12:07 +01001612 test::ScopedKeyValueConfig field_trials_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001613 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001614 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001615 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001616 int codec_width_;
1617 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001618 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001619 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001620 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001621 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001622 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001623 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001624 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001625 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001626};
1627
mflodmancc3d4422017-08-03 08:27:51 -07001628TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001629 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001630 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001631 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001632 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001633 WaitForEncodedFrame(1);
Markus Handell2cfc1af2022-08-19 08:16:48 +00001634 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
mflodmancc3d4422017-08-03 08:27:51 -07001635 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001636}
1637
mflodmancc3d4422017-08-03 08:27:51 -07001638TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001639 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001640 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001641 // The encoder will cache up to one frame for a short duration. Adding two
1642 // frames means that the first frame will be dropped and the second frame will
1643 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001644 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001645 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001646 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Markus Handell28c71802021-11-08 10:11:55 +01001647 AdvanceTime(TimeDelta::Zero());
Markus Handell2cfc1af2022-08-19 08:16:48 +00001648 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
perkj26091b12016-09-01 01:17:40 -07001649
Henrik Boström381d1092020-05-12 18:49:07 +02001650 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001651 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001652
Sebastian Janssona3177052018-04-10 13:05:49 +02001653 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001654 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001655 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1656
1657 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001658 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001659}
1660
mflodmancc3d4422017-08-03 08:27:51 -07001661TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001662 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001663 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001664 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001665 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001666
Henrik Boström381d1092020-05-12 18:49:07 +02001667 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001668 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1669
Sebastian Janssona3177052018-04-10 13:05:49 +02001670 // The encoder will cache up to one frame for a short duration. Adding two
1671 // frames means that the first frame will be dropped and the second frame will
1672 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001673 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001674 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001675
Henrik Boström381d1092020-05-12 18:49:07 +02001676 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001677 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001678 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001679 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1680 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001681 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001682}
1683
mflodmancc3d4422017-08-03 08:27:51 -07001684TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001685 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001686 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001687 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001688 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001689
1690 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001691 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001692
perkja49cbd32016-09-16 07:53:41 -07001693 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001694 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001695 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001696}
1697
mflodmancc3d4422017-08-03 08:27:51 -07001698TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001699 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001700 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001701
perkja49cbd32016-09-16 07:53:41 -07001702 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001703 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001704
mflodmancc3d4422017-08-03 08:27:51 -07001705 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001706 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001707 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001708 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
Markus Handell2cfc1af2022-08-19 08:16:48 +00001709 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
perkj26091b12016-09-01 01:17:40 -07001710}
1711
Markus Handell9a478b52021-11-18 16:07:01 +01001712TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
1713 test::FrameForwarder source;
1714 video_stream_encoder_->SetSource(&source,
1715 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02001716 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001717 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001718
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001719 int dropped_count = 0;
1720 stats_proxy_->SetDroppedFrameCallback(
1721 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1722 ++dropped_count;
1723 });
1724
Markus Handell9a478b52021-11-18 16:07:01 +01001725 source.IncomingCapturedFrame(CreateFrame(1, nullptr));
1726 source.IncomingCapturedFrame(CreateFrame(2, nullptr));
1727 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001728 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001729 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001730}
1731
Henrik Boström56db9ff2021-03-24 09:06:45 +01001732TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001733 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001734 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001735
1736 rtc::Event frame_destroyed_event;
1737 video_source_.IncomingCapturedFrame(
1738 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001739 WaitForEncodedFrame(1);
1740 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1741 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001742 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1743 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001744 video_stream_encoder_->Stop();
1745}
1746
Henrik Boström56db9ff2021-03-24 09:06:45 +01001747TEST_F(VideoStreamEncoderTest,
1748 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001749 // Use the cropping factory.
1750 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001751 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001752 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1753 kMaxPayloadLength);
1754 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1755
1756 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001757 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001758 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001759 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1760 WaitForEncodedFrame(1);
1761 // The encoder will have been configured once.
1762 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001763 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1764 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 06:59:12 -07001765
1766 // Now send in a fake frame that needs to be cropped as the width/height
1767 // aren't divisible by 4 (see CreateEncoderStreams above).
1768 rtc::Event frame_destroyed_event;
1769 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1770 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001771 WaitForEncodedFrame(2);
1772 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1773 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001774 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1775 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001776 video_stream_encoder_->Stop();
1777}
1778
Evan Shrubsole895556e2020-10-05 09:15:13 +02001779TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1780 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001781 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001782
1783 video_source_.IncomingCapturedFrame(
1784 CreateNV12Frame(1, codec_width_, codec_height_));
1785 WaitForEncodedFrame(1);
1786 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1787 fake_encoder_.GetLastInputPixelFormat());
1788 video_stream_encoder_->Stop();
1789}
1790
Henrik Boström56db9ff2021-03-24 09:06:45 +01001791TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001792 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001793 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001794
1795 fake_encoder_.SetPreferredPixelFormats({});
1796
1797 rtc::Event frame_destroyed_event;
1798 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1799 1, &frame_destroyed_event, codec_width_, codec_height_));
1800 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001801 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001802 fake_encoder_.GetLastInputPixelFormat());
1803 video_stream_encoder_->Stop();
1804}
1805
Henrik Boström56db9ff2021-03-24 09:06:45 +01001806TEST_F(VideoStreamEncoderTest,
1807 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001808 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001809 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001810
1811 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1812
1813 rtc::Event frame_destroyed_event;
1814 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1815 1, &frame_destroyed_event, codec_width_, codec_height_));
1816 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001817 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001818 fake_encoder_.GetLastInputPixelFormat());
1819 video_stream_encoder_->Stop();
1820}
1821
Henrik Boström56db9ff2021-03-24 09:06:45 +01001822TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001823 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001824 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001825
1826 // Fake NV12 native frame does not allow mapping to I444.
1827 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1828
1829 rtc::Event frame_destroyed_event;
1830 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1831 1, &frame_destroyed_event, codec_width_, codec_height_));
1832 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001833 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001834 fake_encoder_.GetLastInputPixelFormat());
1835 video_stream_encoder_->Stop();
1836}
1837
Henrik Boström56db9ff2021-03-24 09:06:45 +01001838TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001839 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001840 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001841
1842 rtc::Event frame_destroyed_event;
1843 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1844 1, &frame_destroyed_event, codec_width_, codec_height_));
1845 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001846 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001847 fake_encoder_.GetLastInputPixelFormat());
1848 video_stream_encoder_->Stop();
1849}
1850
Ying Wang9b881ab2020-02-07 14:29:32 +01001851TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001852 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001853 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001854 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1855 WaitForEncodedFrame(1);
1856
Henrik Boström381d1092020-05-12 18:49:07 +02001857 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001858 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001859 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1860 // frames. Adding two frames means that the first frame will be dropped and
1861 // the second frame will be sent to the encoder.
1862 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1863 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1864 WaitForEncodedFrame(3);
1865 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1866 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1867 WaitForEncodedFrame(5);
1868 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1869 video_stream_encoder_->Stop();
1870}
1871
mflodmancc3d4422017-08-03 08:27:51 -07001872TEST_F(VideoStreamEncoderTest,
1873 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001874 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001875 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001876 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001877
1878 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001879 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001880 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001881 // The encoder will have been configured once when the first frame is
1882 // received.
1883 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001884
1885 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001886 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001887 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001888 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001889 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001890
1891 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001892 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001893 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001894 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001895 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001896
mflodmancc3d4422017-08-03 08:27:51 -07001897 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001898}
1899
mflodmancc3d4422017-08-03 08:27:51 -07001900TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001901 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001902 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001903
1904 // Capture a frame and wait for it to synchronize with the encoder thread.
1905 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001906 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001907 // The encoder will have been configured once.
1908 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001909 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1910 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-02 23:45:26 -07001911
1912 codec_width_ *= 2;
1913 codec_height_ *= 2;
1914 // Capture a frame with a higher resolution and wait for it to synchronize
1915 // with the encoder thread.
1916 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001917 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 10:07:11 +02001918 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1919 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 21:37:57 +01001920 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001921
mflodmancc3d4422017-08-03 08:27:51 -07001922 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001923}
1924
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001925TEST_F(VideoStreamEncoderTest,
1926 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001927 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001928 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001929
1930 // Capture a frame and wait for it to synchronize with the encoder thread.
1931 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1932 WaitForEncodedFrame(1);
1933
1934 VideoEncoderConfig video_encoder_config;
1935 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1936 // Changing the max payload data length recreates encoder.
1937 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1938 kMaxPayloadLength / 2);
1939
1940 // Capture a frame and wait for it to synchronize with the encoder thread.
1941 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1942 WaitForEncodedFrame(2);
1943 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1944
1945 video_stream_encoder_->Stop();
1946}
1947
Sergey Silkin5ee69672019-07-02 14:18:34 +02001948TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001949 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001950 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001951
1952 VideoEncoderConfig video_encoder_config;
1953 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02001954 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1955 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001956 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1957 kMaxPayloadLength);
1958
1959 // Capture a frame and wait for it to synchronize with the encoder thread.
1960 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1961 WaitForEncodedFrame(1);
1962 // The encoder will have been configured once when the first frame is
1963 // received.
1964 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001965 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001966 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001967 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001968 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1969
Sergey Silkin6456e352019-07-08 17:56:40 +02001970 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1971 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 10:07:11 +02001972 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1973 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001974 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1975 kMaxPayloadLength);
1976
1977 // Capture a frame and wait for it to synchronize with the encoder thread.
1978 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1979 WaitForEncodedFrame(2);
1980 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1981 // Bitrate limits have changed - rate allocator should be reconfigured,
1982 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 10:07:11 +02001983 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001984 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001985 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001986 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001987 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001988
1989 video_stream_encoder_->Stop();
1990}
1991
Sergey Silkin6456e352019-07-08 17:56:40 +02001992TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001993 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001994 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001995 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001996
Sergey Silkincd02eba2020-01-20 14:48:40 +01001997 const uint32_t kMinEncBitrateKbps = 100;
1998 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001999 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01002000 /*frame_size_pixels=*/codec_width_ * codec_height_,
2001 /*min_start_bitrate_bps=*/0,
2002 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
2003 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002004 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2005
Sergey Silkincd02eba2020-01-20 14:48:40 +01002006 VideoEncoderConfig video_encoder_config;
2007 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2008 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
2009 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2010 (kMinEncBitrateKbps + 1) * 1000;
2011 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2012 kMaxPayloadLength);
2013
2014 // When both encoder and app provide bitrate limits, the intersection of
2015 // provided sets should be used.
2016 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2017 WaitForEncodedFrame(1);
2018 EXPECT_EQ(kMaxEncBitrateKbps,
2019 bitrate_allocator_factory_.codec_config().maxBitrate);
2020 EXPECT_EQ(kMinEncBitrateKbps + 1,
2021 bitrate_allocator_factory_.codec_config().minBitrate);
2022
2023 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
2024 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2025 (kMinEncBitrateKbps - 1) * 1000;
2026 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2027 kMaxPayloadLength);
2028 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002029 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002030 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002031 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002032 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002033 bitrate_allocator_factory_.codec_config().minBitrate);
2034
Sergey Silkincd02eba2020-01-20 14:48:40 +01002035 video_stream_encoder_->Stop();
2036}
2037
2038TEST_F(VideoStreamEncoderTest,
2039 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02002040 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002041 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002042
2043 const uint32_t kMinAppBitrateKbps = 100;
2044 const uint32_t kMaxAppBitrateKbps = 200;
2045 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
2046 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
2047 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2048 /*frame_size_pixels=*/codec_width_ * codec_height_,
2049 /*min_start_bitrate_bps=*/0,
2050 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
2051 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
2052 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2053
2054 VideoEncoderConfig video_encoder_config;
2055 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2056 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
2057 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2058 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002059 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2060 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002061
Sergey Silkincd02eba2020-01-20 14:48:40 +01002062 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2063 WaitForEncodedFrame(1);
2064 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002065 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002066 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002067 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02002068
2069 video_stream_encoder_->Stop();
2070}
2071
2072TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002073 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02002074 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002075 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02002076
2077 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002078 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002079 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002080 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002081 fake_encoder_.SetResolutionBitrateLimits(
2082 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
2083
2084 VideoEncoderConfig video_encoder_config;
2085 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2086 video_encoder_config.max_bitrate_bps = 0;
2087 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2088 kMaxPayloadLength);
2089
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002090 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002091 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2092 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002093 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2094 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002095 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2096 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2097
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002098 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002099 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2100 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002101 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2102 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002103 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2104 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2105
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002106 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02002107 // encoder for 360p should be used.
2108 video_source_.IncomingCapturedFrame(
2109 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2110 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002111 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2112 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002113 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2114 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2115
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002116 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02002117 // ignored.
2118 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2119 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002120 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2121 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002122 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2123 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002124 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2125 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002126 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2127 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2128
2129 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2130 // for 270p should be used.
2131 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2132 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002133 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2134 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002135 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2136 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2137
2138 video_stream_encoder_->Stop();
2139}
2140
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002141TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002142 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002143 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002144
2145 VideoEncoderConfig video_encoder_config;
2146 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2147 video_encoder_config.max_bitrate_bps = 0;
2148 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2149 kMaxPayloadLength);
2150
2151 // Encode 720p frame to get the default encoder target bitrate.
2152 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2153 WaitForEncodedFrame(1);
2154 const uint32_t kDefaultTargetBitrateFor720pKbps =
2155 bitrate_allocator_factory_.codec_config()
2156 .simulcastStream[0]
2157 .targetBitrate;
2158
2159 // Set the max recommended encoder bitrate to something lower than the default
2160 // target bitrate.
2161 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2162 1280 * 720, 10 * 1000, 10 * 1000,
2163 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2164 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2165
2166 // Change resolution to trigger encoder reinitialization.
2167 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2168 WaitForEncodedFrame(2);
2169 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2170 WaitForEncodedFrame(3);
2171
2172 // Ensure the target bitrate is capped by the max bitrate.
2173 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2174 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2175 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2176 .simulcastStream[0]
2177 .targetBitrate *
2178 1000,
2179 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2180
2181 video_stream_encoder_->Stop();
2182}
2183
Åsa Perssona7e34d32021-01-20 15:36:13 +01002184TEST_F(VideoStreamEncoderTest,
2185 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2186 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2187 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2188 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2189 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2190 fake_encoder_.SetResolutionBitrateLimits(
2191 {kEncoderLimits270p, kEncoderLimits360p});
2192
2193 // Two streams, highest stream active.
2194 VideoEncoderConfig config;
Jonas Oreland80c87d72022-09-29 15:01:09 +02002195 webrtc::VideoEncoder::EncoderInfo encoder_info;
Åsa Perssona7e34d32021-01-20 15:36:13 +01002196 const int kNumStreams = 2;
2197 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2198 config.max_bitrate_bps = 0;
2199 config.simulcast_layers[0].active = false;
2200 config.simulcast_layers[1].active = true;
2201 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002202 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002203 "VP8", /*max qp*/ 56, /*screencast*/ false,
Jonas Oreland80c87d72022-09-29 15:01:09 +02002204 /*screenshare enabled*/ false, encoder_info);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002205 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2206
2207 // The encoder bitrate limits for 270p should be used.
2208 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
Erik Språngf449af82022-08-08 17:54:55 +02002209 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002210 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002211 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002212 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002213 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002214 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002215
2216 // The encoder bitrate limits for 360p should be used.
2217 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
Erik Språngf449af82022-08-08 17:54:55 +02002218 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002219 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002220 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002221 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002222 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002223
2224 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2225 video_source_.IncomingCapturedFrame(
2226 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
Erik Språngf449af82022-08-08 17:54:55 +02002227 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002228 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002229 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002230 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002231 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002232
2233 // Resolution higher than 360p. Encoder limits should be ignored.
2234 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02002235 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002236 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002237 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002238 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002239 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002240 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002241 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002242 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002243 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002244
2245 // Resolution lower than 270p. The encoder limits for 270p should be used.
2246 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
Erik Språngf449af82022-08-08 17:54:55 +02002247 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002248 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002249 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002250 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002251 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002252
2253 video_stream_encoder_->Stop();
2254}
2255
2256TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002257 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2258 // Two streams, highest stream active.
2259 VideoEncoderConfig config;
Jonas Oreland80c87d72022-09-29 15:01:09 +02002260 webrtc::VideoEncoder::EncoderInfo encoder_info;
Åsa Persson258e9892021-02-25 10:39:51 +01002261 const int kNumStreams = 2;
2262 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2263 config.max_bitrate_bps = 0;
2264 config.simulcast_layers[0].active = false;
2265 config.simulcast_layers[1].active = true;
2266 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002267 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002268 "VP8", /*max qp*/ 56, /*screencast*/ false,
Jonas Oreland80c87d72022-09-29 15:01:09 +02002269 /*screenshare enabled*/ false, encoder_info);
Åsa Persson258e9892021-02-25 10:39:51 +01002270 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2271
2272 // Default bitrate limits for 270p should be used.
2273 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2274 kDefaultLimits270p =
2275 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002276 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002277 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
Erik Språngf449af82022-08-08 17:54:55 +02002278 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002279 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 10:39:51 +01002280 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002281 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002282 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002283 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002284
2285 // Default bitrate limits for 360p should be used.
2286 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2287 kDefaultLimits360p =
2288 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002289 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002290 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
Erik Språngf449af82022-08-08 17:54:55 +02002291 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Persson258e9892021-02-25 10:39:51 +01002292 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002293 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002294 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002295 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002296
2297 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2298 video_source_.IncomingCapturedFrame(
2299 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
Erik Språngf449af82022-08-08 17:54:55 +02002300 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Persson258e9892021-02-25 10:39:51 +01002301 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002302 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002303 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002304 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002305
2306 // Default bitrate limits for 540p should be used.
2307 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2308 kDefaultLimits540p =
2309 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002310 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002311 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02002312 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Persson258e9892021-02-25 10:39:51 +01002313 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002314 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002315 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002316 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002317
2318 video_stream_encoder_->Stop();
2319}
2320
2321TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002322 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2323 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2324 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2325 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2326 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2327 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2328 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2329 fake_encoder_.SetResolutionBitrateLimits(
2330 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2331
2332 // Three streams, middle stream active.
2333 VideoEncoderConfig config;
Jonas Oreland80c87d72022-09-29 15:01:09 +02002334 webrtc::VideoEncoder::EncoderInfo encoder_info;
Åsa Perssona7e34d32021-01-20 15:36:13 +01002335 const int kNumStreams = 3;
2336 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2337 config.simulcast_layers[0].active = false;
2338 config.simulcast_layers[1].active = true;
2339 config.simulcast_layers[2].active = false;
2340 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002341 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002342 "VP8", /*max qp*/ 56, /*screencast*/ false,
Jonas Oreland80c87d72022-09-29 15:01:09 +02002343 /*screenshare enabled*/ false, encoder_info);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002344 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2345
2346 // The encoder bitrate limits for 360p should be used.
2347 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02002348 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002349 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002350 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002351 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002352 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002353 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002354
2355 // The encoder bitrate limits for 270p should be used.
2356 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02002357 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002358 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002359 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002360 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002361 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002362
2363 video_stream_encoder_->Stop();
2364}
2365
2366TEST_F(VideoStreamEncoderTest,
2367 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2368 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2369 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2370 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2371 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2372 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2373 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2374 fake_encoder_.SetResolutionBitrateLimits(
2375 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2376
2377 // Three streams, lowest stream active.
2378 VideoEncoderConfig config;
Jonas Oreland80c87d72022-09-29 15:01:09 +02002379 webrtc::VideoEncoder::EncoderInfo encoder_info;
Åsa Perssona7e34d32021-01-20 15:36:13 +01002380 const int kNumStreams = 3;
2381 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2382 config.simulcast_layers[0].active = true;
2383 config.simulcast_layers[1].active = false;
2384 config.simulcast_layers[2].active = false;
2385 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002386 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002387 "VP8", /*max qp*/ 56, /*screencast*/ false,
Jonas Oreland80c87d72022-09-29 15:01:09 +02002388 /*screenshare enabled*/ false, encoder_info);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002389 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2390
2391 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2392 // on lowest stream, limits for 270p should not be used
2393 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02002394 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002395 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002396 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002397 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002398 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002399 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002400
2401 video_stream_encoder_->Stop();
2402}
2403
2404TEST_F(VideoStreamEncoderTest,
2405 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2406 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2407 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2408 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2409 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2410 fake_encoder_.SetResolutionBitrateLimits(
2411 {kEncoderLimits270p, kEncoderLimits360p});
2412 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2413
2414 // Two streams, highest stream active.
2415 VideoEncoderConfig config;
Jonas Oreland80c87d72022-09-29 15:01:09 +02002416 webrtc::VideoEncoder::EncoderInfo encoder_info;
Åsa Perssona7e34d32021-01-20 15:36:13 +01002417 const int kNumStreams = 2;
2418 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2419 config.simulcast_layers[0].active = false;
2420 config.simulcast_layers[1].active = true;
2421 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2422 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002423 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002424 "VP8", /*max qp*/ 56, /*screencast*/ false,
Jonas Oreland80c87d72022-09-29 15:01:09 +02002425 /*screenshare enabled*/ false, encoder_info);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002426 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2427
2428 // The encoder bitrate limits for 270p should be used.
2429 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
Erik Språngf449af82022-08-08 17:54:55 +02002430 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002431 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002432 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002433 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002434 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002435 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002436
2437 // The max configured bitrate is less than the encoder limit for 360p.
2438 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
Erik Språngf449af82022-08-08 17:54:55 +02002439 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002440 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002441 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002442 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002443 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002444
2445 video_stream_encoder_->Stop();
2446}
2447
mflodmancc3d4422017-08-03 08:27:51 -07002448TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002449 EXPECT_TRUE(video_source_.has_sinks());
2450 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002451 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002452 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002453 EXPECT_FALSE(video_source_.has_sinks());
2454 EXPECT_TRUE(new_video_source.has_sinks());
2455
mflodmancc3d4422017-08-03 08:27:51 -07002456 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002457}
2458
mflodmancc3d4422017-08-03 08:27:51 -07002459TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002460 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002461 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002462 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002463 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002464}
2465
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002466class ResolutionAlignmentTest
2467 : public VideoStreamEncoderTest,
2468 public ::testing::WithParamInterface<
2469 ::testing::tuple<int, std::vector<double>>> {
2470 public:
2471 ResolutionAlignmentTest()
2472 : requested_alignment_(::testing::get<0>(GetParam())),
2473 scale_factors_(::testing::get<1>(GetParam())) {}
2474
2475 protected:
Chunbo Hua46ad2512022-12-06 23:56:33 +08002476 const uint32_t requested_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002477 const std::vector<double> scale_factors_;
2478};
2479
2480INSTANTIATE_TEST_SUITE_P(
2481 AlignmentAndScaleFactors,
2482 ResolutionAlignmentTest,
2483 ::testing::Combine(
2484 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2485 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2486 std::vector<double>{-1.0, -1.0},
2487 std::vector<double>{-1.0, -1.0, -1.0},
2488 std::vector<double>{4.0, 2.0, 1.0},
2489 std::vector<double>{9999.0, -1.0, 1.0},
2490 std::vector<double>{3.99, 2.01, 1.0},
2491 std::vector<double>{4.9, 1.7, 1.25},
2492 std::vector<double>{10.0, 4.0, 3.0},
2493 std::vector<double>{1.75, 3.5},
2494 std::vector<double>{1.5, 2.5},
2495 std::vector<double>{1.3, 1.0})));
2496
2497TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2498 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002499 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002500 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2501 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2502
2503 // Fill config with the scaling factor by which to reduce encoding size.
2504 const int num_streams = scale_factors_.size();
2505 VideoEncoderConfig config;
Jonas Oreland80c87d72022-09-29 15:01:09 +02002506 webrtc::VideoEncoder::EncoderInfo encoder_info;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002507 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2508 for (int i = 0; i < num_streams; ++i) {
2509 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2510 }
2511 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002512 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002513 "VP8", /*max qp*/ 56, /*screencast*/ false,
Jonas Oreland80c87d72022-09-29 15:01:09 +02002514 /*screenshare enabled*/ false, encoder_info);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002515 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2516
Henrik Boström381d1092020-05-12 18:49:07 +02002517 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002518 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2519 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002520 // Wait for all layers before triggering event.
2521 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002522
2523 // On the 1st frame, we should have initialized the encoder and
2524 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002525 int64_t timestamp_ms = kFrameIntervalMs;
2526 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2527 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002528 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002529
2530 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2531 // (It's up the to the encoder to potentially drop the previous frame,
2532 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002533 timestamp_ms += kFrameIntervalMs;
2534 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2535 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002536 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002537
Asa Persson606d3cb2021-10-04 10:07:11 +02002538 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002539 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2540 // Frame size should be a multiple of the requested alignment.
2541 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
Chunbo Hua46ad2512022-12-06 23:56:33 +08002542 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0u);
2543 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0u);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002544 // Aspect ratio should match.
2545 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2546 codec.height * codec.simulcastStream[i].width);
2547 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002548
2549 video_stream_encoder_->Stop();
2550}
2551
Jonathan Yubc771b72017-12-08 17:04:29 -08002552TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2553 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002554 const int kWidth = 1280;
2555 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002556
2557 // We rely on the automatic resolution adaptation, but we handle framerate
2558 // adaptation manually by mocking the stats proxy.
2559 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002560
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002561 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002562 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002563 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002564 video_stream_encoder_->SetSource(&video_source_,
2565 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002566 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002567 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002568 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002569 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2570
Jonathan Yubc771b72017-12-08 17:04:29 -08002571 // Adapt down as far as possible.
2572 rtc::VideoSinkWants last_wants;
2573 int64_t t = 1;
2574 int loop_count = 0;
2575 do {
2576 ++loop_count;
2577 last_wants = video_source_.sink_wants();
2578
2579 // Simulate the framerate we've been asked to adapt to.
2580 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2581 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2582 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2583 mock_stats.input_frame_rate = fps;
2584 stats_proxy_->SetMockStats(mock_stats);
2585
2586 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2587 sink_.WaitForEncodedFrame(t);
2588 t += frame_interval_ms;
2589
mflodmancc3d4422017-08-03 08:27:51 -07002590 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002591 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002592 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002593 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2594 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002595 } while (video_source_.sink_wants().max_pixel_count <
2596 last_wants.max_pixel_count ||
2597 video_source_.sink_wants().max_framerate_fps <
2598 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002599
Jonathan Yubc771b72017-12-08 17:04:29 -08002600 // Verify that we've adapted all the way down.
2601 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002602 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002603 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2604 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002605 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002606 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2607 *video_source_.last_sent_height());
2608 EXPECT_EQ(kMinBalancedFramerateFps,
2609 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002610
Jonathan Yubc771b72017-12-08 17:04:29 -08002611 // Adapt back up the same number of times we adapted down.
2612 for (int i = 0; i < loop_count - 1; ++i) {
2613 last_wants = video_source_.sink_wants();
2614
2615 // Simulate the framerate we've been asked to adapt to.
2616 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2617 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2618 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2619 mock_stats.input_frame_rate = fps;
2620 stats_proxy_->SetMockStats(mock_stats);
2621
2622 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2623 sink_.WaitForEncodedFrame(t);
2624 t += frame_interval_ms;
2625
Henrik Boström91aa7322020-04-28 12:24:33 +02002626 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002627 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002628 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002629 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2630 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002631 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2632 last_wants.max_pixel_count ||
2633 video_source_.sink_wants().max_framerate_fps >
2634 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002635 }
2636
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002637 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002638 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002639 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002640 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2641 EXPECT_EQ((loop_count - 1) * 2,
2642 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002643
mflodmancc3d4422017-08-03 08:27:51 -07002644 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002645}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002646
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002647TEST_F(VideoStreamEncoderTest,
2648 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02002649 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2650 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002651 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002652
2653 const int kFrameWidth = 1280;
2654 const int kFrameHeight = 720;
2655
2656 int64_t ntp_time = kFrameIntervalMs;
2657
2658 // Force an input frame rate to be available, or the adaptation call won't
2659 // know what framerate to adapt form.
2660 const int kInputFps = 30;
2661 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2662 stats.input_frame_rate = kInputFps;
2663 stats_proxy_->SetMockStats(stats);
2664
2665 video_source_.set_adaptation_enabled(true);
2666 video_stream_encoder_->SetSource(
2667 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002668 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002669 video_source_.IncomingCapturedFrame(
2670 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2671 sink_.WaitForEncodedFrame(ntp_time);
2672 ntp_time += kFrameIntervalMs;
2673
2674 // Trigger CPU overuse.
2675 video_stream_encoder_->TriggerCpuOveruse();
2676 video_source_.IncomingCapturedFrame(
2677 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2678 sink_.WaitForEncodedFrame(ntp_time);
2679 ntp_time += kFrameIntervalMs;
2680
2681 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2682 EXPECT_EQ(std::numeric_limits<int>::max(),
2683 video_source_.sink_wants().max_pixel_count);
2684 // Some framerate constraint should be set.
2685 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2686 EXPECT_LT(restricted_fps, kInputFps);
2687 video_source_.IncomingCapturedFrame(
2688 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2689 sink_.WaitForEncodedFrame(ntp_time);
2690 ntp_time += 100;
2691
Henrik Boström2671dac2020-05-19 16:29:09 +02002692 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002693 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2694 // Give the encoder queue time to process the change in degradation preference
2695 // by waiting for an encoded frame.
2696 video_source_.IncomingCapturedFrame(
2697 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2698 sink_.WaitForEncodedFrame(ntp_time);
2699 ntp_time += kFrameIntervalMs;
2700
2701 video_stream_encoder_->TriggerQualityLow();
2702 video_source_.IncomingCapturedFrame(
2703 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2704 sink_.WaitForEncodedFrame(ntp_time);
2705 ntp_time += kFrameIntervalMs;
2706
2707 // Some resolution constraint should be set.
2708 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2709 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2710 kFrameWidth * kFrameHeight);
2711 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2712
2713 int pixel_count = video_source_.sink_wants().max_pixel_count;
2714 // Triggering a CPU underuse should not change the sink wants since it has
2715 // not been overused for resolution since we changed degradation preference.
2716 video_stream_encoder_->TriggerCpuUnderuse();
2717 video_source_.IncomingCapturedFrame(
2718 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2719 sink_.WaitForEncodedFrame(ntp_time);
2720 ntp_time += kFrameIntervalMs;
2721 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2722 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2723
Evan Shrubsole64469032020-06-11 10:45:29 +02002724 // Change the degradation preference back. CPU underuse should not adapt since
2725 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002726 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002727 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2728 video_source_.IncomingCapturedFrame(
2729 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2730 sink_.WaitForEncodedFrame(ntp_time);
2731 ntp_time += 100;
2732 // Resolution adaptations is gone after changing degradation preference.
2733 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2734 EXPECT_EQ(std::numeric_limits<int>::max(),
2735 video_source_.sink_wants().max_pixel_count);
2736 // The fps adaptation from above is now back.
2737 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2738
2739 // Trigger CPU underuse.
2740 video_stream_encoder_->TriggerCpuUnderuse();
2741 video_source_.IncomingCapturedFrame(
2742 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2743 sink_.WaitForEncodedFrame(ntp_time);
2744 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002745 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2746
2747 // Trigger QP underuse, fps should return to normal.
2748 video_stream_encoder_->TriggerQualityHigh();
2749 video_source_.IncomingCapturedFrame(
2750 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2751 sink_.WaitForEncodedFrame(ntp_time);
2752 ntp_time += kFrameIntervalMs;
2753 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002754
2755 video_stream_encoder_->Stop();
2756}
2757
mflodmancc3d4422017-08-03 08:27:51 -07002758TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002759 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002760 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002761 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002762
sprangc5d62e22017-04-02 23:53:04 -07002763 const int kFrameWidth = 1280;
2764 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002765
Åsa Persson8c1bf952018-09-13 10:42:19 +02002766 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002767
kthelgason5e13d412016-12-01 03:59:51 -08002768 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002769 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002770 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002771 frame_timestamp += kFrameIntervalMs;
2772
perkj803d97f2016-11-01 11:45:46 -07002773 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002774 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002775 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002776 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002777 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002778 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002779
asapersson0944a802017-04-07 00:57:58 -07002780 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002781 // wanted resolution.
2782 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2783 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2784 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002785 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002786
2787 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002788 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002789 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002790 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002791 // Give the encoder queue time to process the change in degradation preference
2792 // by waiting for an encoded frame.
2793 new_video_source.IncomingCapturedFrame(
2794 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2795 sink_.WaitForEncodedFrame(frame_timestamp);
2796 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002797 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002798 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002799
sprangc5d62e22017-04-02 23:53:04 -07002800 // Force an input frame rate to be available, or the adaptation call won't
2801 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002802 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002803 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002804 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002805 stats_proxy_->SetMockStats(stats);
2806
mflodmancc3d4422017-08-03 08:27:51 -07002807 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002808 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002809 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002810 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002811 frame_timestamp += kFrameIntervalMs;
2812
2813 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002814 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002815 EXPECT_EQ(std::numeric_limits<int>::max(),
2816 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002817 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002818
asapersson02465b82017-04-10 01:12:52 -07002819 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002820 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2821 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002822 // Give the encoder queue time to process the change in degradation preference
2823 // by waiting for an encoded frame.
2824 new_video_source.IncomingCapturedFrame(
2825 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2826 sink_.WaitForEncodedFrame(frame_timestamp);
2827 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002828 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002829
mflodmancc3d4422017-08-03 08:27:51 -07002830 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002831 new_video_source.IncomingCapturedFrame(
2832 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002833 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002834 frame_timestamp += kFrameIntervalMs;
2835
2836 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002837 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002838
2839 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002840 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002841 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002842 // Give the encoder queue time to process the change in degradation preference
2843 // by waiting for an encoded frame.
2844 new_video_source.IncomingCapturedFrame(
2845 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2846 sink_.WaitForEncodedFrame(frame_timestamp);
2847 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002848 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2849 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002850 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002851 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002852
2853 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002854 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002855 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002856 // Give the encoder queue time to process the change in degradation preference
2857 // by waiting for an encoded frame.
2858 new_video_source.IncomingCapturedFrame(
2859 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2860 sink_.WaitForEncodedFrame(frame_timestamp);
2861 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002862 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2863 EXPECT_EQ(std::numeric_limits<int>::max(),
2864 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002865 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002866
mflodmancc3d4422017-08-03 08:27:51 -07002867 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002868}
2869
mflodmancc3d4422017-08-03 08:27:51 -07002870TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002871 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002872 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002873
asaperssonfab67072017-04-04 05:51:49 -07002874 const int kWidth = 1280;
2875 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002876 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002877 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002878 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2879 EXPECT_FALSE(stats.bw_limited_resolution);
2880 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2881
2882 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002883 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002884 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002885 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002886
2887 stats = stats_proxy_->GetStats();
2888 EXPECT_TRUE(stats.bw_limited_resolution);
2889 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2890
2891 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002892 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002893 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002894 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002895
2896 stats = stats_proxy_->GetStats();
2897 EXPECT_FALSE(stats.bw_limited_resolution);
2898 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2899 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2900
mflodmancc3d4422017-08-03 08:27:51 -07002901 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002902}
2903
mflodmancc3d4422017-08-03 08:27:51 -07002904TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002905 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002906 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002907
2908 const int kWidth = 1280;
2909 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002910 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002911 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002912 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2913 EXPECT_FALSE(stats.cpu_limited_resolution);
2914 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2915
2916 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002917 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002918 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002919 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002920
2921 stats = stats_proxy_->GetStats();
2922 EXPECT_TRUE(stats.cpu_limited_resolution);
2923 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2924
2925 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002926 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002927 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002928 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002929
2930 stats = stats_proxy_->GetStats();
2931 EXPECT_FALSE(stats.cpu_limited_resolution);
2932 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002933 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002934
mflodmancc3d4422017-08-03 08:27:51 -07002935 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002936}
2937
mflodmancc3d4422017-08-03 08:27:51 -07002938TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002939 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002940 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002941
asaperssonfab67072017-04-04 05:51:49 -07002942 const int kWidth = 1280;
2943 const int kHeight = 720;
2944 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002945 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002946 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002947 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002948 EXPECT_FALSE(stats.cpu_limited_resolution);
2949 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2950
asaperssonfab67072017-04-04 05:51:49 -07002951 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002952 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002953 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002954 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002955 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002956 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002957 EXPECT_TRUE(stats.cpu_limited_resolution);
2958 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2959
2960 // Set new source with adaptation still enabled.
2961 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002962 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002963 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002964
asaperssonfab67072017-04-04 05:51:49 -07002965 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002966 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002967 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002968 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002969 EXPECT_TRUE(stats.cpu_limited_resolution);
2970 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2971
2972 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002973 video_stream_encoder_->SetSource(&new_video_source,
2974 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002975
asaperssonfab67072017-04-04 05:51:49 -07002976 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002977 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002978 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002979 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002980 EXPECT_FALSE(stats.cpu_limited_resolution);
2981 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2982
2983 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002984 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002985 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002986
asaperssonfab67072017-04-04 05:51:49 -07002987 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002988 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002989 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002990 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002991 EXPECT_TRUE(stats.cpu_limited_resolution);
2992 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2993
asaperssonfab67072017-04-04 05:51:49 -07002994 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002995 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002996 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002997 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002998 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002999 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08003000 EXPECT_FALSE(stats.cpu_limited_resolution);
3001 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003002 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003003
mflodmancc3d4422017-08-03 08:27:51 -07003004 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003005}
3006
mflodmancc3d4422017-08-03 08:27:51 -07003007TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02003008 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003009 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003010
asaperssonfab67072017-04-04 05:51:49 -07003011 const int kWidth = 1280;
3012 const int kHeight = 720;
3013 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003014 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003015 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003016 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003017 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003018 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003019
3020 // Set new source with adaptation still enabled.
3021 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003022 video_stream_encoder_->SetSource(&new_video_source,
3023 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08003024
asaperssonfab67072017-04-04 05:51:49 -07003025 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003026 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08003027 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003028 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003029 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003030 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003031
asaperssonfab67072017-04-04 05:51:49 -07003032 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003033 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003034 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003035 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003036 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003037 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003038 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003039 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003040
asaperssonfab67072017-04-04 05:51:49 -07003041 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003042 video_stream_encoder_->SetSource(&new_video_source,
3043 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08003044
asaperssonfab67072017-04-04 05:51:49 -07003045 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003046 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003047 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003048 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003049 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003050 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003051
asapersson02465b82017-04-10 01:12:52 -07003052 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07003053 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003054 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003055
asaperssonfab67072017-04-04 05:51:49 -07003056 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003057 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08003058 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003059 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003060 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003061 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
3062 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003063
mflodmancc3d4422017-08-03 08:27:51 -07003064 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003065}
3066
mflodmancc3d4422017-08-03 08:27:51 -07003067TEST_F(VideoStreamEncoderTest,
3068 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02003069 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003070 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07003071
3072 const int kWidth = 1280;
3073 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02003074 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07003075 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003076 video_source_.IncomingCapturedFrame(
3077 CreateFrame(timestamp_ms, kWidth, kHeight));
3078 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003079 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3080 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3081 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3082
3083 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003084 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003085 timestamp_ms += kFrameIntervalMs;
3086 video_source_.IncomingCapturedFrame(
3087 CreateFrame(timestamp_ms, kWidth, kHeight));
3088 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003089 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3090 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3091 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3092
3093 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07003094 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003095 timestamp_ms += kFrameIntervalMs;
3096 video_source_.IncomingCapturedFrame(
3097 CreateFrame(timestamp_ms, kWidth, kHeight));
3098 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003099 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3100 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3101 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3102
Niels Möller4db138e2018-04-19 09:04:13 +02003103 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07003104 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003105
3106 VideoEncoderConfig video_encoder_config;
3107 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3108 // Make format different, to force recreation of encoder.
3109 video_encoder_config.video_format.parameters["foo"] = "foo";
3110 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003111 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003112 timestamp_ms += kFrameIntervalMs;
3113 video_source_.IncomingCapturedFrame(
3114 CreateFrame(timestamp_ms, kWidth, kHeight));
3115 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003116 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3117 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3118 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3119
mflodmancc3d4422017-08-03 08:27:51 -07003120 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003121}
3122
mflodmancc3d4422017-08-03 08:27:51 -07003123TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003124 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003125 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003126 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003127
3128 const int kWidth = 1280;
3129 const int kHeight = 720;
3130 int sequence = 1;
3131
3132 // Enable BALANCED preference, no initial limitation.
3133 test::FrameForwarder source;
3134 video_stream_encoder_->SetSource(&source,
3135 webrtc::DegradationPreference::BALANCED);
3136 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3137 WaitForEncodedFrame(sequence++);
3138 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3139 EXPECT_FALSE(stats.cpu_limited_resolution);
3140 EXPECT_FALSE(stats.cpu_limited_framerate);
3141 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3142
3143 // Trigger CPU overuse, should now adapt down.
3144 video_stream_encoder_->TriggerCpuOveruse();
3145 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3146 WaitForEncodedFrame(sequence++);
3147 stats = stats_proxy_->GetStats();
3148 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3149
3150 // Set new degradation preference should clear restrictions since we changed
3151 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003152 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003153 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3154 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3155 WaitForEncodedFrame(sequence++);
3156 stats = stats_proxy_->GetStats();
3157 EXPECT_FALSE(stats.cpu_limited_resolution);
3158 EXPECT_FALSE(stats.cpu_limited_framerate);
3159 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3160
3161 // Force an input frame rate to be available, or the adaptation call won't
3162 // know what framerate to adapt from.
3163 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3164 mock_stats.input_frame_rate = 30;
3165 stats_proxy_->SetMockStats(mock_stats);
3166 video_stream_encoder_->TriggerCpuOveruse();
3167 stats_proxy_->ResetMockStats();
3168 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3169 WaitForEncodedFrame(sequence++);
3170
3171 // We have now adapted once.
3172 stats = stats_proxy_->GetStats();
3173 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3174
3175 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003176 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3177 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003178 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3179 WaitForEncodedFrame(sequence++);
3180 stats = stats_proxy_->GetStats();
3181 EXPECT_FALSE(stats.cpu_limited_resolution);
3182 EXPECT_FALSE(stats.cpu_limited_framerate);
3183 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3184
3185 video_stream_encoder_->Stop();
3186}
3187
3188TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003189 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003190 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003191 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003192
asapersson0944a802017-04-07 00:57:58 -07003193 const int kWidth = 1280;
3194 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003195 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003196
asaperssonfab67072017-04-04 05:51:49 -07003197 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003198 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003199 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003200 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003201 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003202 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3203
asapersson02465b82017-04-10 01:12:52 -07003204 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003205 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003206 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003207 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003208 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003209 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003210 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003211 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3212
3213 // Set new source with adaptation still enabled.
3214 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003215 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003216 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003217
3218 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003219 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003220 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003221 stats = stats_proxy_->GetStats();
3222 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003223 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003224 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3225
sprangc5d62e22017-04-02 23:53:04 -07003226 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003227 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003228 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003229 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003230 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003231 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003232 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003233 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003234 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003235 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003236 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3237
sprangc5d62e22017-04-02 23:53:04 -07003238 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003239 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003240 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3241 mock_stats.input_frame_rate = 30;
3242 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003243 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003244 stats_proxy_->ResetMockStats();
3245
3246 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003247 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003248 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003249
3250 // Framerate now adapted.
3251 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003252 EXPECT_FALSE(stats.cpu_limited_resolution);
3253 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003254 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3255
3256 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003257 video_stream_encoder_->SetSource(&new_video_source,
3258 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003259 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003260 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003261 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003262
3263 stats = stats_proxy_->GetStats();
3264 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003265 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003266 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3267
3268 // Try to trigger overuse. Should not succeed.
3269 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003270 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003271 stats_proxy_->ResetMockStats();
3272
3273 stats = stats_proxy_->GetStats();
3274 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003275 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003276 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3277
3278 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003279 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003280 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003281 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003282 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003283 stats = stats_proxy_->GetStats();
3284 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003285 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003286 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003287
3288 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003289 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003290 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003291 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003292 stats = stats_proxy_->GetStats();
3293 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003294 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003295 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3296
3297 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003298 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003299 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003300 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003301 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003302 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003303 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003304 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003305 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003306 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003307 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3308
3309 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003310 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003311 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003312 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003313 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003314 stats = stats_proxy_->GetStats();
3315 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003316 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003317 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003318 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003319
mflodmancc3d4422017-08-03 08:27:51 -07003320 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003321}
3322
mflodmancc3d4422017-08-03 08:27:51 -07003323TEST_F(VideoStreamEncoderTest,
3324 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003325 const int kWidth = 1280;
3326 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003327 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003328 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003329
asaperssonfab67072017-04-04 05:51:49 -07003330 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003331 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003332
asaperssonfab67072017-04-04 05:51:49 -07003333 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003334 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003335
asaperssonfab67072017-04-04 05:51:49 -07003336 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003337 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003338
asaperssonfab67072017-04-04 05:51:49 -07003339 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003340 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003341
kthelgason876222f2016-11-29 01:44:11 -08003342 // Expect a scale down.
3343 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003344 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003345
asapersson02465b82017-04-10 01:12:52 -07003346 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003347 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003348 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003349 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003350
asaperssonfab67072017-04-04 05:51:49 -07003351 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003352 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003353 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003354 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003355
asaperssonfab67072017-04-04 05:51:49 -07003356 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003357 EXPECT_EQ(std::numeric_limits<int>::max(),
3358 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003359
asaperssonfab67072017-04-04 05:51:49 -07003360 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003361 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003362 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003363 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003364
asapersson02465b82017-04-10 01:12:52 -07003365 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003366 EXPECT_EQ(std::numeric_limits<int>::max(),
3367 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003368
mflodmancc3d4422017-08-03 08:27:51 -07003369 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003370}
3371
mflodmancc3d4422017-08-03 08:27:51 -07003372TEST_F(VideoStreamEncoderTest,
3373 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003374 const int kWidth = 1280;
3375 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003376 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003377 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003378
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003379 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003380 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003381 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003382 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003383
3384 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003385 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003386 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003387 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3388 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3389
3390 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003391 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003392 EXPECT_THAT(source.sink_wants(),
3393 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003394 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3395 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3396 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3397
3398 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003399 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003400 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3401 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3402 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3403
mflodmancc3d4422017-08-03 08:27:51 -07003404 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003405}
3406
mflodmancc3d4422017-08-03 08:27:51 -07003407TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003408 const int kWidth = 1280;
3409 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003410 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003411 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003412
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003413 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003414 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003415 video_stream_encoder_->SetSource(&source,
3416 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003417 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3418 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003419 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003420
3421 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003422 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003423 EXPECT_THAT(source.sink_wants(),
3424 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003425 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3426 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3427 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3428
3429 // Trigger adapt down for same input resolution, expect no change.
3430 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3431 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003432 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003433 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3434 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3435 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3436
3437 // Trigger adapt down for larger input resolution, expect no change.
3438 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3439 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003440 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003441 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3442 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3443 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3444
mflodmancc3d4422017-08-03 08:27:51 -07003445 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003446}
3447
mflodmancc3d4422017-08-03 08:27:51 -07003448TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003449 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3450 const int kWidth = 640;
3451 const int kHeight = 360;
3452 const int64_t kFrameIntervalMs = 150;
3453 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003454 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003455
3456 // Enable BALANCED preference, no initial limitation.
3457 AdaptingFrameForwarder source(&time_controller_);
3458 source.set_adaptation_enabled(true);
3459 video_stream_encoder_->SetSource(&source,
3460 webrtc::DegradationPreference::BALANCED);
3461
3462 int64_t timestamp_ms = kFrameIntervalMs;
3463 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3464 sink_.WaitForEncodedFrame(kWidth, kHeight);
3465 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3466 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3467 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3468 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3469
3470 // Trigger adapt down, expect reduced fps (640x360@15fps).
3471 video_stream_encoder_->TriggerQualityLow();
3472 timestamp_ms += kFrameIntervalMs;
3473 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3474 sink_.WaitForEncodedFrame(timestamp_ms);
3475 EXPECT_THAT(source.sink_wants(),
3476 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3477 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3478 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3479 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3480
3481 // Source requests 270p, expect reduced resolution (480x270@15fps).
3482 source.OnOutputFormatRequest(480, 270);
3483 timestamp_ms += kFrameIntervalMs;
3484 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3485 WaitForEncodedFrame(480, 270);
3486 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3487
3488 // Trigger adapt down, expect reduced fps (480x270@10fps).
3489 video_stream_encoder_->TriggerQualityLow();
3490 timestamp_ms += kFrameIntervalMs;
3491 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3492 sink_.WaitForEncodedFrame(timestamp_ms);
3493 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3494 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3495 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3496 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3497
3498 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3499 source.OnOutputFormatRequest(320, 180);
3500 timestamp_ms += kFrameIntervalMs;
3501 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3502 WaitForEncodedFrame(320, 180);
3503 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3504
3505 // Trigger adapt down, expect reduced fps (320x180@7fps).
3506 video_stream_encoder_->TriggerQualityLow();
3507 timestamp_ms += kFrameIntervalMs;
3508 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3509 sink_.WaitForEncodedFrame(timestamp_ms);
3510 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3511 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3512 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3513 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3514
3515 // Source requests VGA, expect increased resolution (640x360@7fps).
3516 source.OnOutputFormatRequest(640, 360);
3517 timestamp_ms += kFrameIntervalMs;
3518 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3519 WaitForEncodedFrame(timestamp_ms);
3520 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3521
3522 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3523 video_stream_encoder_->TriggerQualityHigh();
3524 timestamp_ms += kFrameIntervalMs;
3525 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3526 WaitForEncodedFrame(timestamp_ms);
3527 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3528 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3529 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3530 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3531
3532 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3533 video_stream_encoder_->TriggerQualityHigh();
3534 timestamp_ms += kFrameIntervalMs;
3535 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3536 WaitForEncodedFrame(timestamp_ms);
3537 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3538 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3539 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3540 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3541
3542 // Trigger adapt up, expect increased fps (640x360@maxfps).
3543 video_stream_encoder_->TriggerQualityHigh();
3544 timestamp_ms += kFrameIntervalMs;
3545 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3546 WaitForEncodedFrame(timestamp_ms);
3547 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3548 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3549 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3550 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3551
3552 video_stream_encoder_->Stop();
3553}
3554
3555TEST_F(VideoStreamEncoderTest,
3556 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3557 const int kWidth = 1280;
3558 const int kHeight = 720;
3559 const int64_t kFrameIntervalMs = 150;
3560 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003561 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003562
3563 // Enable BALANCED preference, no initial limitation.
3564 AdaptingFrameForwarder source(&time_controller_);
3565 source.set_adaptation_enabled(true);
3566 video_stream_encoder_->SetSource(&source,
3567 webrtc::DegradationPreference::BALANCED);
3568
3569 int64_t timestamp_ms = kFrameIntervalMs;
3570 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3571 sink_.WaitForEncodedFrame(kWidth, kHeight);
3572 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3573 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3574 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3575 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3576
3577 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3578 video_stream_encoder_->TriggerQualityLow();
3579 timestamp_ms += kFrameIntervalMs;
3580 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3581 sink_.WaitForEncodedFrame(timestamp_ms);
3582 EXPECT_THAT(source.sink_wants(),
3583 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3584 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3585 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3586 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3587
3588 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3589 video_stream_encoder_->TriggerQualityLow();
3590 timestamp_ms += kFrameIntervalMs;
3591 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3592 sink_.WaitForEncodedFrame(timestamp_ms);
3593 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3594 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3595 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3596 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3597
3598 // Trigger adapt down, expect reduced fps (640x360@15fps).
3599 video_stream_encoder_->TriggerQualityLow();
3600 timestamp_ms += kFrameIntervalMs;
3601 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3602 WaitForEncodedFrame(timestamp_ms);
3603 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3604 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3605 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3606 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3607
3608 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3609 source.OnOutputFormatRequest(320, 180);
3610 timestamp_ms += kFrameIntervalMs;
3611 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3612 WaitForEncodedFrame(320, 180);
3613 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3614 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3615
3616 // Trigger adapt down, expect reduced fps (320x180@7fps).
3617 video_stream_encoder_->TriggerCpuOveruse();
3618 timestamp_ms += kFrameIntervalMs;
3619 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3620 WaitForEncodedFrame(timestamp_ms);
3621 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3622 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3623 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3624 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3625 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3626 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3627 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3628
3629 // Source requests HD, expect increased resolution (640x360@7fps).
3630 source.OnOutputFormatRequest(1280, 720);
3631 timestamp_ms += kFrameIntervalMs;
3632 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3633 WaitForEncodedFrame(timestamp_ms);
3634 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3635 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3636
3637 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3638 video_stream_encoder_->TriggerCpuUnderuse();
3639 timestamp_ms += kFrameIntervalMs;
3640 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3641 WaitForEncodedFrame(timestamp_ms);
3642 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3643 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3644 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3645 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3646 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3647 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3648 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3649
3650 // Trigger adapt up, expect increased fps (640x360@maxfps).
3651 video_stream_encoder_->TriggerQualityHigh();
3652 video_stream_encoder_->TriggerCpuUnderuse();
3653 timestamp_ms += kFrameIntervalMs;
3654 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3655 WaitForEncodedFrame(timestamp_ms);
3656 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3657 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3658 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3659 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3660 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3661 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3662 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3663
3664 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3665 video_stream_encoder_->TriggerQualityHigh();
3666 video_stream_encoder_->TriggerCpuUnderuse();
3667 timestamp_ms += kFrameIntervalMs;
3668 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3669 WaitForEncodedFrame(timestamp_ms);
3670 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3671 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3672 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3673 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3674 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3675 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3676 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3677
3678 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3679 video_stream_encoder_->TriggerQualityHigh();
3680 video_stream_encoder_->TriggerCpuUnderuse();
3681 timestamp_ms += kFrameIntervalMs;
3682 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3683 WaitForEncodedFrame(timestamp_ms);
3684 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3685 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3686 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3687 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3688 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3689 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3690 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3691
3692 video_stream_encoder_->Stop();
3693}
3694
3695TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003696 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003697 const int kWidth = 1280;
3698 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003699 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003700 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003701
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003702 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003703 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003704 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003705 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003706
3707 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003708 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003709 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003710 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3711 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3712
3713 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003714 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003715 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003716 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3717 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3718
mflodmancc3d4422017-08-03 08:27:51 -07003719 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003720}
3721
mflodmancc3d4422017-08-03 08:27:51 -07003722TEST_F(VideoStreamEncoderTest,
3723 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003724 const int kWidth = 1280;
3725 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003726 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003727 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003728
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003729 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003730 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003731 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003732 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003733
3734 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003735 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003736 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003737 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003738 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3739
3740 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003741 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003742 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003743 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003744 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3745
mflodmancc3d4422017-08-03 08:27:51 -07003746 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003747}
3748
mflodmancc3d4422017-08-03 08:27:51 -07003749TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003750 const int kWidth = 1280;
3751 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003752 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003753 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003754
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003755 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003756 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003757 video_stream_encoder_->SetSource(&source,
3758 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003759
3760 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3761 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003762 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003763 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3764 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3765 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3766
3767 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003768 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003769 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003770 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3771 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3772 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3773
mflodmancc3d4422017-08-03 08:27:51 -07003774 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003775}
3776
mflodmancc3d4422017-08-03 08:27:51 -07003777TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003778 const int kWidth = 1280;
3779 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003780 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003781 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003782
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003783 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003784 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003785 video_stream_encoder_->SetSource(&source,
3786 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003787
3788 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3789 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003790 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003791 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3792 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3793 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3794
3795 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003796 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003797 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003798 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3799 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3800 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3801
mflodmancc3d4422017-08-03 08:27:51 -07003802 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003803}
3804
mflodmancc3d4422017-08-03 08:27:51 -07003805TEST_F(VideoStreamEncoderTest,
3806 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003807 const int kWidth = 1280;
3808 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003809 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003810 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003811
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003812 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003813 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003814 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003815 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003816 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003817
3818 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003819 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003820 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003821 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3822 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3823
3824 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003825 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003826 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003827 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003828 EXPECT_THAT(source.sink_wants(),
3829 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003830 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3831 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3832
3833 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003834 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003835 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003836 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3837 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3838 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3839
mflodmancc3d4422017-08-03 08:27:51 -07003840 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003841}
3842
mflodmancc3d4422017-08-03 08:27:51 -07003843TEST_F(VideoStreamEncoderTest,
3844 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003845 const int kWidth = 1280;
3846 const int kHeight = 720;
3847 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003848 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003849 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003850
3851 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3852 stats.input_frame_rate = kInputFps;
3853 stats_proxy_->SetMockStats(stats);
3854
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003855 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003856 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3857 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003858 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003859
3860 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003861 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003862 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3863 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003864 EXPECT_THAT(video_source_.sink_wants(),
3865 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003866
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003867 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003868 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003869 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003870 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003871 // Give the encoder queue time to process the change in degradation preference
3872 // by waiting for an encoded frame.
3873 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3874 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003875 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003876
3877 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003878 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003879 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3880 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003881 EXPECT_THAT(new_video_source.sink_wants(),
3882 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003883
3884 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003885 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003886 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003887
mflodmancc3d4422017-08-03 08:27:51 -07003888 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003889}
3890
mflodmancc3d4422017-08-03 08:27:51 -07003891TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003892 const int kWidth = 1280;
3893 const int kHeight = 720;
3894 const size_t kNumFrames = 10;
3895
Henrik Boström381d1092020-05-12 18:49:07 +02003896 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003897 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003898
asaperssond0de2952017-04-21 01:47:31 -07003899 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003900 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003901 video_source_.set_adaptation_enabled(true);
3902
3903 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3904 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3905
3906 int downscales = 0;
3907 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003908 video_source_.IncomingCapturedFrame(
3909 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3910 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003911
asaperssonfab67072017-04-04 05:51:49 -07003912 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003913 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003914 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003915 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003916
3917 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3918 ++downscales;
3919
3920 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3921 EXPECT_EQ(downscales,
3922 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3923 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003924 }
mflodmancc3d4422017-08-03 08:27:51 -07003925 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003926}
3927
mflodmancc3d4422017-08-03 08:27:51 -07003928TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003929 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3930 const int kWidth = 1280;
3931 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003932 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003933 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003934
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003935 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003936 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003937 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003938 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003939 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003940
Åsa Persson8c1bf952018-09-13 10:42:19 +02003941 int64_t timestamp_ms = kFrameIntervalMs;
3942 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003943 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003944 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003945 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3946 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3947
3948 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003949 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003950 timestamp_ms += kFrameIntervalMs;
3951 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3952 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003953 EXPECT_THAT(source.sink_wants(),
3954 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003955 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3956 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3957
3958 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003959 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003960 timestamp_ms += kFrameIntervalMs;
3961 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003962 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003963 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003964 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3965 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3966
3967 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003968 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003969 timestamp_ms += kFrameIntervalMs;
3970 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3971 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003972 EXPECT_THAT(source.sink_wants(),
3973 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003974 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3975 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3976
3977 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003978 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003979 timestamp_ms += kFrameIntervalMs;
3980 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003981 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003982 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003983 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3984 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3985
mflodmancc3d4422017-08-03 08:27:51 -07003986 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003987}
3988
mflodmancc3d4422017-08-03 08:27:51 -07003989TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003990 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3991 const int kWidth = 1280;
3992 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003993 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003994 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003995
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003996 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003997 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003998 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003999 video_stream_encoder_->SetSource(&source,
4000 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004001
Åsa Persson8c1bf952018-09-13 10:42:19 +02004002 int64_t timestamp_ms = kFrameIntervalMs;
4003 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07004004 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004005 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004006 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4007 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4008
4009 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07004010 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004011 timestamp_ms += kFrameIntervalMs;
4012 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4013 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004014 EXPECT_THAT(source.sink_wants(),
4015 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004016 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4017 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4018
4019 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07004020 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004021 timestamp_ms += kFrameIntervalMs;
4022 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07004023 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004024 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004025 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4026 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4027
4028 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07004029 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004030 timestamp_ms += kFrameIntervalMs;
4031 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4032 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004033 EXPECT_THAT(source.sink_wants(),
4034 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004035 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4036 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4037
4038 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07004039 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004040 timestamp_ms += kFrameIntervalMs;
4041 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07004042 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004043 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004044 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4045 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4046
mflodmancc3d4422017-08-03 08:27:51 -07004047 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004048}
4049
Sergey Silkin41c650b2019-10-14 13:12:19 +02004050TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
4051 fake_encoder_.SetResolutionBitrateLimits(
4052 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4053
Henrik Boström381d1092020-05-12 18:49:07 +02004054 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004055 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4056 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4057 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4058 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004059
4060 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004061 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004062 source.set_adaptation_enabled(true);
4063 video_stream_encoder_->SetSource(
4064 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4065
4066 // Insert 720p frame.
4067 int64_t timestamp_ms = kFrameIntervalMs;
4068 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4069 WaitForEncodedFrame(1280, 720);
4070
4071 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02004072 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004073 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4074 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4075 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4076 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004077 video_stream_encoder_->TriggerQualityLow();
4078
4079 // Insert 720p frame. It should be downscaled and encoded.
4080 timestamp_ms += kFrameIntervalMs;
4081 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4082 WaitForEncodedFrame(960, 540);
4083
4084 // Trigger adapt up. Higher resolution should not be requested duo to lack
4085 // of bitrate.
4086 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004087 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02004088
4089 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02004090 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004091 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4092 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4093 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4094 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004095
4096 // Trigger adapt up. Higher resolution should be requested.
4097 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004098 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02004099
4100 video_stream_encoder_->Stop();
4101}
4102
4103TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
4104 fake_encoder_.SetResolutionBitrateLimits(
4105 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4106
4107 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02004108 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004109 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4110 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4111 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4112 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004113
4114 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004115 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004116 source.set_adaptation_enabled(true);
4117 video_stream_encoder_->SetSource(
4118 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4119
4120 // Insert 720p frame. It should be dropped and lower resolution should be
4121 // requested.
4122 int64_t timestamp_ms = kFrameIntervalMs;
4123 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4124 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004125 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004126
4127 // Insert 720p frame. It should be downscaled and encoded.
4128 timestamp_ms += kFrameIntervalMs;
4129 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4130 WaitForEncodedFrame(960, 540);
4131
4132 video_stream_encoder_->Stop();
4133}
4134
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004135class BalancedDegradationTest : public VideoStreamEncoderTest {
4136 protected:
4137 void SetupTest() {
4138 // Reset encoder for field trials to take effect.
4139 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02004140 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004141
4142 // Enable BALANCED preference.
4143 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02004144 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4145 }
4146
Asa Persson606d3cb2021-10-04 10:07:11 +02004147 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02004148 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004149 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004150 }
4151
Åsa Persson45b176f2019-09-30 11:19:05 +02004152 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004153 timestamp_ms_ += kFrameIntervalMs;
4154 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02004155 }
4156
4157 void InsertFrameAndWaitForEncoded() {
4158 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004159 sink_.WaitForEncodedFrame(timestamp_ms_);
4160 }
4161
4162 const int kWidth = 640; // pixels:640x360=230400
4163 const int kHeight = 360;
4164 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4165 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004166 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004167};
4168
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004169TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004170 test::ScopedKeyValueConfig field_trials(
4171 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004172 "WebRTC-Video-BalancedDegradationSettings/"
4173 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4174 SetupTest();
4175
4176 // Force input frame rate.
4177 const int kInputFps = 24;
4178 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4179 stats.input_frame_rate = kInputFps;
4180 stats_proxy_->SetMockStats(stats);
4181
Åsa Persson45b176f2019-09-30 11:19:05 +02004182 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004183 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004184
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004185 // Trigger adapt down, expect scaled down framerate and resolution,
4186 // since Fps diff (input-requested:0) < threshold.
4187 video_stream_encoder_->TriggerQualityLow();
4188 EXPECT_THAT(source_.sink_wants(),
4189 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004190
4191 video_stream_encoder_->Stop();
4192}
4193
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004194TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004195 test::ScopedKeyValueConfig field_trials(
4196 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004197 "WebRTC-Video-BalancedDegradationSettings/"
4198 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4199 SetupTest();
4200
4201 // Force input frame rate.
4202 const int kInputFps = 25;
4203 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4204 stats.input_frame_rate = kInputFps;
4205 stats_proxy_->SetMockStats(stats);
4206
Åsa Persson45b176f2019-09-30 11:19:05 +02004207 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004208 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004209
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004210 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4211 // Fps diff (input-requested:1) == threshold.
4212 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004213 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004214
4215 video_stream_encoder_->Stop();
4216}
4217
4218TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004219 test::ScopedKeyValueConfig field_trials(
4220 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004221 "WebRTC-Video-BalancedDegradationSettings/"
4222 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4223 SetupTest();
4224
4225 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4226
Åsa Persson45b176f2019-09-30 11:19:05 +02004227 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004228 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004229
4230 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4231 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004232 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004233
4234 video_stream_encoder_->Stop();
4235}
4236
Åsa Perssonccfb3402019-09-25 15:13:04 +02004237TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004238 test::ScopedKeyValueConfig field_trials(
4239 field_trials_,
Åsa Persson1b247f12019-08-14 17:26:39 +02004240 "WebRTC-Video-BalancedDegradationSettings/"
4241 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004242 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004243
Asa Persson606d3cb2021-10-04 10:07:11 +02004244 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4245 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4246 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004247
Åsa Persson45b176f2019-09-30 11:19:05 +02004248 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004249 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004250 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4251
4252 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4253 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004254 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004255 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004256 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4257
4258 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4259 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004260 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004261 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004262 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4263
Åsa Persson30ab0152019-08-27 12:22:33 +02004264 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4265 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004266 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004267 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004268 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004269 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4270
4271 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004272 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004273 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004274 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004275
Åsa Persson30ab0152019-08-27 12:22:33 +02004276 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004277 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004278 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004279 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004280 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004281 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4282
4283 video_stream_encoder_->Stop();
4284}
4285
Åsa Perssonccfb3402019-09-25 15:13:04 +02004286TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004287 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004288 test::ScopedKeyValueConfig field_trials(
4289 field_trials_,
Åsa Persson45b176f2019-09-30 11:19:05 +02004290 "WebRTC-Video-BalancedDegradationSettings/"
4291 "pixels:57600|129600|230400,fps:7|24|24/");
4292 SetupTest();
Asa Persson606d3cb2021-10-04 10:07:11 +02004293 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 11:19:05 +02004294
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004295 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004296
4297 // Insert frame, expect scaled down:
4298 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4299 InsertFrame();
Markus Handell2cfc1af2022-08-19 08:16:48 +00004300 EXPECT_FALSE(WaitForFrame(TimeDelta::Seconds(1)));
Åsa Persson45b176f2019-09-30 11:19:05 +02004301 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4302 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4303
4304 // Insert frame, expect scaled down:
4305 // resolution (320x180@24fps).
4306 InsertFrame();
Markus Handell2cfc1af2022-08-19 08:16:48 +00004307 EXPECT_FALSE(WaitForFrame(TimeDelta::Seconds(1)));
Åsa Persson45b176f2019-09-30 11:19:05 +02004308 EXPECT_LT(source_.sink_wants().max_pixel_count,
4309 source_.last_wants().max_pixel_count);
4310 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4311
4312 // Frame should not be dropped (min pixels per frame reached).
4313 InsertFrameAndWaitForEncoded();
4314
4315 video_stream_encoder_->Stop();
4316}
4317
4318TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004319 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004320 test::ScopedKeyValueConfig field_trials(
4321 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004322 "WebRTC-Video-BalancedDegradationSettings/"
4323 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004324 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004325
Asa Persson606d3cb2021-10-04 10:07:11 +02004326 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4327 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4328 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004329
Åsa Persson45b176f2019-09-30 11:19:05 +02004330 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004331 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004332 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4333
4334 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4335 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004336 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004337 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004338 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4339
4340 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4341 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004342 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004343 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004344 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4345
4346 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4347 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004348 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004349 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004350 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4351
Åsa Persson30ab0152019-08-27 12:22:33 +02004352 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4353 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004354 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004355 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004356 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4357
4358 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4359 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004360 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004361 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4362
4363 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004364 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004365 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004366 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004367 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004368 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4369
4370 video_stream_encoder_->Stop();
4371}
4372
Åsa Perssonccfb3402019-09-25 15:13:04 +02004373TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004374 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004375 test::ScopedKeyValueConfig field_trials(
4376 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004377 "WebRTC-Video-BalancedDegradationSettings/"
4378 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004379 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004380
Asa Persson606d3cb2021-10-04 10:07:11 +02004381 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4382 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4383 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4384 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4385 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004386
Åsa Persson45b176f2019-09-30 11:19:05 +02004387 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004388 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004389 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4390
4391 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4392 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004393 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004394 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004395 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4396
4397 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4398 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004399 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004400 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004401 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4402
4403 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4404 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004405 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004406 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004407 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4408
4409 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4410 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004411 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004412 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4413
4414 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004415 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004416 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004417 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004418 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004419 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4420
4421 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004422 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004423 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004424 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004425 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4426
4427 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004428 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004429 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004430 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004431 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004432 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4433
Åsa Persson1b247f12019-08-14 17:26:39 +02004434 video_stream_encoder_->Stop();
4435}
4436
mflodmancc3d4422017-08-03 08:27:51 -07004437TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004438 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4439 const int kWidth = 1280;
4440 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004441 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004442 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004443
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004444 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004445 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004446 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004447 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004448 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004449
Åsa Persson8c1bf952018-09-13 10:42:19 +02004450 int64_t timestamp_ms = kFrameIntervalMs;
4451 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004452 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004453 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004454 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4455 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4456 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4457 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4458
4459 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004460 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004461 timestamp_ms += kFrameIntervalMs;
4462 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4463 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004464 EXPECT_THAT(source.sink_wants(),
4465 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004466 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4467 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4468 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4469 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4470
4471 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004472 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004473 timestamp_ms += kFrameIntervalMs;
4474 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4475 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004476 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004477 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4478 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4479 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4480 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4481
Jonathan Yubc771b72017-12-08 17:04:29 -08004482 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004483 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004484 timestamp_ms += kFrameIntervalMs;
4485 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4486 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004487 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004488 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4489 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004490 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004491 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4492
Jonathan Yubc771b72017-12-08 17:04:29 -08004493 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004494 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004495 timestamp_ms += kFrameIntervalMs;
4496 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4497 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004498 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004499 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004500 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4501 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4502 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4503 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4504
Jonathan Yubc771b72017-12-08 17:04:29 -08004505 // Trigger quality adapt down, expect no change (min resolution reached).
4506 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004507 timestamp_ms += kFrameIntervalMs;
4508 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4509 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004510 EXPECT_THAT(source.sink_wants(), FpsMax());
4511 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004512 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4513 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4514 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4515 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4516
Evan Shrubsole64469032020-06-11 10:45:29 +02004517 // Trigger quality adapt up, expect upscaled resolution (480x270).
4518 video_stream_encoder_->TriggerQualityHigh();
4519 timestamp_ms += kFrameIntervalMs;
4520 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4521 WaitForEncodedFrame(timestamp_ms);
4522 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4523 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4524 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4525 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4526 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4527
4528 // Trigger quality and cpu adapt up since both are most limited, expect
4529 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004530 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004531 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004532 timestamp_ms += kFrameIntervalMs;
4533 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4534 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004535 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004536 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4537 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4538 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004539 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004540
Evan Shrubsole64469032020-06-11 10:45:29 +02004541 // Trigger quality and cpu adapt up since both are most limited, expect
4542 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004543 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004544 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004545 timestamp_ms += kFrameIntervalMs;
4546 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4547 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004548 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004549 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004550 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004551 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004552 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4553 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004554
Evan Shrubsole64469032020-06-11 10:45:29 +02004555 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4556 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004557 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004558 timestamp_ms += kFrameIntervalMs;
4559 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4560 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004561 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004562 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4563 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004564 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004565 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004566
4567 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004568 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004569 timestamp_ms += kFrameIntervalMs;
4570 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004571 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004572 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004573 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004574 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4575 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004576 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004577 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004578
mflodmancc3d4422017-08-03 08:27:51 -07004579 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004580}
4581
mflodmancc3d4422017-08-03 08:27:51 -07004582TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004583 const int kWidth = 640;
4584 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004585
Henrik Boström381d1092020-05-12 18:49:07 +02004586 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004587 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004588
perkj803d97f2016-11-01 11:45:46 -07004589 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004590 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004591 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004592 }
4593
mflodmancc3d4422017-08-03 08:27:51 -07004594 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004595 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004596 video_source_.IncomingCapturedFrame(CreateFrame(
4597 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004598 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004599 }
4600
mflodmancc3d4422017-08-03 08:27:51 -07004601 video_stream_encoder_->Stop();
4602 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004603 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004604
Ying Wangef3998f2019-12-09 13:06:53 +01004605 EXPECT_METRIC_EQ(
4606 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4607 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004608 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4609}
4610
mflodmancc3d4422017-08-03 08:27:51 -07004611TEST_F(VideoStreamEncoderTest,
4612 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004613 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004614 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004615 const int kWidth = 640;
4616 const int kHeight = 360;
4617
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004618 video_stream_encoder_->SetSource(&video_source_,
4619 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004620
4621 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4622 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004623 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004624 }
4625
mflodmancc3d4422017-08-03 08:27:51 -07004626 video_stream_encoder_->Stop();
4627 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004628 stats_proxy_.reset();
4629
4630 EXPECT_EQ(0,
4631 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4632}
4633
Per Kjellanderdcef6412020-10-07 15:09:05 +02004634TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4635 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004636 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004637 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004638
4639 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004640 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 10:07:11 +02004641 SimulcastRateAllocator(fake_encoder_.config())
4642 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004643 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004644
Henrik Boström381d1092020-05-12 18:49:07 +02004645 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004646 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004647
sprang57c2fff2017-01-16 06:24:02 -08004648 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004649 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4650 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004651 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4652 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4653
Erik Språngd7329ca2019-02-21 21:19:53 +01004654 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004655 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004656 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004657
Per Kjellanderdcef6412020-10-07 15:09:05 +02004658 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004659 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004660 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4661 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004662 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004663 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004664
Per Kjellanderdcef6412020-10-07 15:09:05 +02004665 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004666 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004667 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004668 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004669 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4670 WaitForEncodedFrame(CurrentTimeMs());
4671 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004672 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004673 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004674
mflodmancc3d4422017-08-03 08:27:51 -07004675 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004676}
4677
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004678TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004679 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004680 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004681 kVideoLayersAllocation);
4682
4683 const int kDefaultFps = 30;
4684
4685 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004686 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004687
4688 video_source_.IncomingCapturedFrame(
4689 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4690 WaitForEncodedFrame(CurrentTimeMs());
4691 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4692 VideoLayersAllocation last_layer_allocation =
4693 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004694 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004695 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4696
4697 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004698 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004699 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 10:07:11 +02004700 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 17:53:22 +02004701 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4702
Erik Språng9d69cbe2020-10-22 17:44:42 +02004703 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004704 int number_of_layers_allocation = 1;
4705 const int64_t start_time_ms = CurrentTimeMs();
4706 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4707 video_source_.IncomingCapturedFrame(
4708 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4709 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004710 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4711 number_of_layers_allocation = sink_.number_of_layers_allocations();
4712 VideoLayersAllocation new_allocation =
4713 sink_.GetLastVideoLayersAllocation();
4714 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4715 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4716 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4717 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4718 .target_bitrate_per_temporal_layer,
4719 last_layer_allocation.active_spatial_layers[0]
4720 .target_bitrate_per_temporal_layer);
4721 last_layer_allocation = new_allocation;
4722 }
4723 }
4724 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4725 video_stream_encoder_->Stop();
4726}
4727
4728TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004729 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004730 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4731 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4732 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004733 VideoEncoderConfig video_encoder_config;
4734 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4735 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004736 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004737 video_encoder_config.content_type =
4738 VideoEncoderConfig::ContentType::kRealtimeVideo;
4739 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004740 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004741 VideoEncoder::GetDefaultVp8Settings());
4742 for (auto& layer : video_encoder_config.simulcast_layers) {
4743 layer.num_temporal_layers = 2;
4744 }
4745 // Simulcast layers are used for enabling/disabling streams.
4746 video_encoder_config.simulcast_layers[0].active = true;
4747 video_encoder_config.simulcast_layers[1].active = false;
4748 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004749 ConfigureEncoder(std::move(video_encoder_config),
4750 VideoStreamEncoder::BitrateAllocationCallbackType::
4751 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004752
4753 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004754 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004755
4756 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4757 WaitForEncodedFrame(CurrentTimeMs());
4758 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4759 VideoLayersAllocation last_layer_allocation =
4760 sink_.GetLastVideoLayersAllocation();
4761
4762 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4763 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4764 .target_bitrate_per_temporal_layer,
4765 SizeIs(2));
4766 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4767 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4768 video_stream_encoder_->Stop();
4769}
4770
4771TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004772 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004773 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4774 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4775 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004776 VideoEncoderConfig video_encoder_config;
4777 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4778 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004779 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004780 video_encoder_config.content_type =
4781 VideoEncoderConfig::ContentType::kRealtimeVideo;
4782 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004783 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004784 VideoEncoder::GetDefaultVp8Settings());
4785 for (auto& layer : video_encoder_config.simulcast_layers) {
4786 layer.num_temporal_layers = 2;
4787 }
4788 // Simulcast layers are used for enabling/disabling streams.
4789 video_encoder_config.simulcast_layers[0].active = true;
4790 video_encoder_config.simulcast_layers[1].active = false;
4791 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004792 ConfigureEncoder(std::move(video_encoder_config),
4793 VideoStreamEncoder::BitrateAllocationCallbackType::
4794 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004795
4796 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004797 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004798
4799 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4800 WaitForEncodedFrame(CurrentTimeMs());
4801 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4802 VideoLayersAllocation last_layer_allocation =
4803 sink_.GetLastVideoLayersAllocation();
4804
4805 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4806 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4807 .target_bitrate_per_temporal_layer,
4808 SizeIs(2));
4809 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4810
4811 video_stream_encoder_->Stop();
4812}
4813
4814TEST_F(VideoStreamEncoderTest,
4815 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4816 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4817 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004818 VideoEncoderConfig video_encoder_config;
4819 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4820 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004821 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004822 video_encoder_config.content_type =
4823 VideoEncoderConfig::ContentType::kRealtimeVideo;
4824 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4825 vp9_settings.numberOfSpatialLayers = 2;
4826 vp9_settings.numberOfTemporalLayers = 2;
4827 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4828 vp9_settings.automaticResizeOn = false;
4829 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004830 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004831 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004832 ConfigureEncoder(std::move(video_encoder_config),
4833 VideoStreamEncoder::BitrateAllocationCallbackType::
4834 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004835
4836 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004837 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004838
4839 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4840 WaitForEncodedFrame(CurrentTimeMs());
4841 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4842 VideoLayersAllocation last_layer_allocation =
4843 sink_.GetLastVideoLayersAllocation();
4844
4845 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4846 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4847 .target_bitrate_per_temporal_layer,
4848 SizeIs(2));
4849 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4850 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4851 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4852 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4853 .target_bitrate_per_temporal_layer,
4854 SizeIs(2));
4855 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4856 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4857 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4858
4859 // Since full SVC is used, expect the top layer to utilize the full target
4860 // rate.
4861 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4862 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004863 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004864 video_stream_encoder_->Stop();
4865}
4866
4867TEST_F(VideoStreamEncoderTest,
4868 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4869 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4870 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004871 VideoEncoderConfig video_encoder_config;
4872 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4873 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004874 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004875 video_encoder_config.content_type =
4876 VideoEncoderConfig::ContentType::kRealtimeVideo;
4877 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4878 vp9_settings.numberOfSpatialLayers = 2;
4879 vp9_settings.numberOfTemporalLayers = 2;
4880 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4881 vp9_settings.automaticResizeOn = false;
4882 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004883 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004884 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004885 ConfigureEncoder(std::move(video_encoder_config),
4886 VideoStreamEncoder::BitrateAllocationCallbackType::
4887 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004888
4889 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004890 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004891
4892 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4893 WaitForEncodedFrame(CurrentTimeMs());
4894 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4895 VideoLayersAllocation last_layer_allocation =
4896 sink_.GetLastVideoLayersAllocation();
4897
4898 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4899 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4900 .target_bitrate_per_temporal_layer,
4901 SizeIs(1));
4902 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4903 .target_bitrate_per_temporal_layer,
4904 SizeIs(1));
4905 // Since full SVC is used, expect the top layer to utilize the full target
4906 // rate.
4907 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4908 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004909 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004910 video_stream_encoder_->Stop();
4911}
4912
4913TEST_F(VideoStreamEncoderTest,
4914 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4915 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4916 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004917 VideoEncoderConfig video_encoder_config;
4918 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4919 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004920 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004921 video_encoder_config.content_type =
4922 VideoEncoderConfig::ContentType::kRealtimeVideo;
4923 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4924 vp9_settings.numberOfSpatialLayers = 2;
4925 vp9_settings.numberOfTemporalLayers = 2;
4926 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4927 vp9_settings.automaticResizeOn = false;
4928 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004929 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004930 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004931 ConfigureEncoder(std::move(video_encoder_config),
4932 VideoStreamEncoder::BitrateAllocationCallbackType::
4933 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004934
4935 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004936 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004937
4938 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4939 WaitForEncodedFrame(CurrentTimeMs());
4940 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4941 VideoLayersAllocation last_layer_allocation =
4942 sink_.GetLastVideoLayersAllocation();
4943
4944 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4945 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4946 .target_bitrate_per_temporal_layer,
4947 SizeIs(2));
4948 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4949 .target_bitrate_per_temporal_layer,
4950 SizeIs(2));
4951 // Since KSVC is, spatial layers are independend except on key frames.
4952 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4953 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004954 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004955 video_stream_encoder_->Stop();
4956}
4957
4958TEST_F(VideoStreamEncoderTest,
4959 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4960 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4961 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4962 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004963 VideoEncoderConfig video_encoder_config;
4964 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4965 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004966 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004967 video_encoder_config.content_type =
4968 VideoEncoderConfig::ContentType::kRealtimeVideo;
4969 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4970 vp9_settings.numberOfSpatialLayers = 3;
4971 vp9_settings.numberOfTemporalLayers = 2;
4972 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4973 vp9_settings.automaticResizeOn = false;
4974 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004975 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004976 vp9_settings);
4977 // Simulcast layers are used for enabling/disabling streams.
4978 video_encoder_config.simulcast_layers.resize(3);
4979 video_encoder_config.simulcast_layers[0].active = false;
4980 video_encoder_config.simulcast_layers[1].active = true;
4981 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004982 ConfigureEncoder(std::move(video_encoder_config),
4983 VideoStreamEncoder::BitrateAllocationCallbackType::
4984 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004985
4986 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004987 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004988
4989 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4990 WaitForEncodedFrame(CurrentTimeMs());
4991 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4992 VideoLayersAllocation last_layer_allocation =
4993 sink_.GetLastVideoLayersAllocation();
4994
4995 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4996 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4997 .target_bitrate_per_temporal_layer,
4998 SizeIs(2));
4999 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
5000 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5001
5002 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
5003 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
5004 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
5005 .target_bitrate_per_temporal_layer,
5006 SizeIs(2));
5007 // Since full SVC is used, expect the top layer to utilize the full target
5008 // rate.
5009 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
5010 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02005011 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005012 video_stream_encoder_->Stop();
5013}
5014
5015TEST_F(VideoStreamEncoderTest,
5016 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
5017 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5018 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5019 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005020 VideoEncoderConfig video_encoder_config;
5021 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5022 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005023 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005024 video_encoder_config.content_type =
5025 VideoEncoderConfig::ContentType::kRealtimeVideo;
5026 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5027 vp9_settings.numberOfSpatialLayers = 3;
5028 vp9_settings.numberOfTemporalLayers = 2;
5029 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5030 vp9_settings.automaticResizeOn = false;
5031 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005032 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005033 vp9_settings);
5034 // Simulcast layers are used for enabling/disabling streams.
5035 video_encoder_config.simulcast_layers.resize(3);
5036 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005037 ConfigureEncoder(std::move(video_encoder_config),
5038 VideoStreamEncoder::BitrateAllocationCallbackType::
5039 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005040
5041 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005042 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005043
5044 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5045 WaitForEncodedFrame(CurrentTimeMs());
5046 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5047 VideoLayersAllocation last_layer_allocation =
5048 sink_.GetLastVideoLayersAllocation();
5049
5050 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
5051 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5052 .target_bitrate_per_temporal_layer,
5053 SizeIs(2));
5054 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
5055 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5056
5057 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
5058 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
5059 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
5060 .target_bitrate_per_temporal_layer,
5061 SizeIs(2));
5062 video_stream_encoder_->Stop();
5063}
5064
5065TEST_F(VideoStreamEncoderTest,
5066 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
5067 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5068 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5069 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005070 VideoEncoderConfig video_encoder_config;
5071 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5072 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005073 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005074 video_encoder_config.content_type =
5075 VideoEncoderConfig::ContentType::kRealtimeVideo;
5076 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5077 vp9_settings.numberOfSpatialLayers = 3;
5078 vp9_settings.numberOfTemporalLayers = 2;
5079 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5080 vp9_settings.automaticResizeOn = false;
5081 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005082 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005083 vp9_settings);
5084 // Simulcast layers are used for enabling/disabling streams.
5085 video_encoder_config.simulcast_layers.resize(3);
5086 video_encoder_config.simulcast_layers[0].active = false;
5087 video_encoder_config.simulcast_layers[1].active = false;
5088 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005089 ConfigureEncoder(std::move(video_encoder_config),
5090 VideoStreamEncoder::BitrateAllocationCallbackType::
5091 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005092
5093 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005094 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005095
5096 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5097 WaitForEncodedFrame(CurrentTimeMs());
5098 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5099 VideoLayersAllocation last_layer_allocation =
5100 sink_.GetLastVideoLayersAllocation();
5101
5102 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5103 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5104 .target_bitrate_per_temporal_layer,
5105 SizeIs(2));
5106 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5107 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5108 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5109 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02005110 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005111 video_stream_encoder_->Stop();
5112}
5113
5114TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5115 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005116 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005117 kVideoLayersAllocation);
5118 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005119 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005120
5121 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5122 WaitForEncodedFrame(CurrentTimeMs());
5123 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5124 VideoLayersAllocation last_layer_allocation =
5125 sink_.GetLastVideoLayersAllocation();
5126
5127 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5128 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5129 .target_bitrate_per_temporal_layer,
5130 SizeIs(1));
5131 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5132 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005133 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005134 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5135 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5136 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5137 video_stream_encoder_->Stop();
5138}
5139
5140TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02005141 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5142 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005143 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02005144 kVideoLayersAllocation);
5145
5146 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005147 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005148
5149 video_source_.IncomingCapturedFrame(
5150 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5151 WaitForEncodedFrame(CurrentTimeMs());
5152 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5153 VideoLayersAllocation last_layer_allocation =
5154 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02005155 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02005156 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5157 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5158 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005159 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 17:53:22 +02005160
5161 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005162 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5163 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005164 video_source_.IncomingCapturedFrame(
5165 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5166 WaitForEncodedFrame(CurrentTimeMs());
5167
5168 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5169 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5170 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5171 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5172 .target_bitrate_per_temporal_layer[0],
5173 DataRate::Zero());
5174
5175 video_stream_encoder_->Stop();
5176}
5177
Per Kjellander4190ce92020-12-15 17:24:55 +01005178TEST_F(VideoStreamEncoderTest,
5179 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5180 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005181 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01005182 kVideoLayersAllocation);
5183
5184 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005185 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5186 0, 0, 0);
Per Kjellander4190ce92020-12-15 17:24:55 +01005187
5188 video_source_.IncomingCapturedFrame(
5189 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5190 WaitForEncodedFrame(CurrentTimeMs());
5191 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5192 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5193 SizeIs(2));
5194 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5195 codec_width_);
5196 EXPECT_EQ(
5197 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5198 codec_height_);
5199
5200 video_source_.IncomingCapturedFrame(
5201 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5202 WaitForEncodedFrame(CurrentTimeMs());
5203 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5204 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5205 SizeIs(2));
5206 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5207 codec_width_ / 2);
5208 EXPECT_EQ(
5209 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5210 codec_height_ / 2);
5211
5212 video_stream_encoder_->Stop();
5213}
5214
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005215TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5216 // 2 TLs configured, temporal layers supported by encoder.
5217 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005218 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005219 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005220 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005221 fake_encoder_.SetTemporalLayersSupported(0, true);
5222
5223 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 10:07:11 +02005224 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005225 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005226 kNumTemporalLayers, /*temporal_id*/ 0,
5227 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005228 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005229 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005230 kNumTemporalLayers, /*temporal_id*/ 1,
5231 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005232 VideoBitrateAllocation expected_bitrate;
5233 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5234 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5235
5236 VerifyAllocatedBitrate(expected_bitrate);
5237 video_stream_encoder_->Stop();
5238}
5239
5240TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5241 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005242 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005243 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005244 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005245 fake_encoder_.SetTemporalLayersSupported(0, false);
5246
5247 // Temporal layers not supported by the encoder.
5248 // Total bitrate should be at ti:0.
5249 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005250 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005251
5252 VerifyAllocatedBitrate(expected_bitrate);
5253 video_stream_encoder_->Stop();
5254}
5255
5256TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005257 webrtc::test::ScopedKeyValueConfig field_trials(
5258 field_trials_,
Per Kjellanderdcef6412020-10-07 15:09:05 +02005259 "WebRTC-Video-QualityScalerSettings/"
5260 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5261 // Reset encoder for field trials to take effect.
5262 ConfigureEncoder(video_encoder_config_.Copy());
5263
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005264 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005265 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005266 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005267 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005268 fake_encoder_.SetTemporalLayersSupported(0, true);
5269 fake_encoder_.SetTemporalLayersSupported(1, false);
5270
5271 const int kS0Bps = 150000;
5272 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005273 kS0Bps *
5274 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5275 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005276 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005277 kS0Bps *
5278 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5279 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005280 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005281 // Temporal layers not supported by si:1.
5282 VideoBitrateAllocation expected_bitrate;
5283 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5284 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5285 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5286
5287 VerifyAllocatedBitrate(expected_bitrate);
5288 video_stream_encoder_->Stop();
5289}
5290
Niels Möller7dc26b72017-12-06 10:27:48 +01005291TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5292 const int kFrameWidth = 1280;
5293 const int kFrameHeight = 720;
5294 const int kFramerate = 24;
5295
Henrik Boström381d1092020-05-12 18:49:07 +02005296 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005297 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005298 test::FrameForwarder source;
5299 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005300 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005301
5302 // Insert a single frame, triggering initial configuration.
5303 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5304 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5305
5306 EXPECT_EQ(
5307 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5308 kDefaultFramerate);
5309
5310 // Trigger reconfigure encoder (without resetting the entire instance).
5311 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005312 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5313 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005314 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005315 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005316 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005317 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5318
5319 // Detector should be updated with fps limit from codec config.
5320 EXPECT_EQ(
5321 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5322 kFramerate);
5323
5324 // Trigger overuse, max framerate should be reduced.
5325 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5326 stats.input_frame_rate = kFramerate;
5327 stats_proxy_->SetMockStats(stats);
5328 video_stream_encoder_->TriggerCpuOveruse();
5329 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5330 int adapted_framerate =
5331 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5332 EXPECT_LT(adapted_framerate, kFramerate);
5333
5334 // Trigger underuse, max framerate should go back to codec configured fps.
5335 // Set extra low fps, to make sure it's actually reset, not just incremented.
5336 stats = stats_proxy_->GetStats();
5337 stats.input_frame_rate = adapted_framerate / 2;
5338 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005339 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005340 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5341 EXPECT_EQ(
5342 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5343 kFramerate);
5344
5345 video_stream_encoder_->Stop();
5346}
5347
5348TEST_F(VideoStreamEncoderTest,
5349 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5350 const int kFrameWidth = 1280;
5351 const int kFrameHeight = 720;
5352 const int kLowFramerate = 15;
5353 const int kHighFramerate = 25;
5354
Henrik Boström381d1092020-05-12 18:49:07 +02005355 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005356 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005357 test::FrameForwarder source;
5358 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005359 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005360
5361 // Trigger initial configuration.
5362 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005363 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5364 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005365 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005366 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005367 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005368 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005369 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5370
5371 EXPECT_EQ(
5372 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5373 kLowFramerate);
5374
5375 // Trigger overuse, max framerate should be reduced.
5376 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5377 stats.input_frame_rate = kLowFramerate;
5378 stats_proxy_->SetMockStats(stats);
5379 video_stream_encoder_->TriggerCpuOveruse();
5380 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5381 int adapted_framerate =
5382 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5383 EXPECT_LT(adapted_framerate, kLowFramerate);
5384
5385 // Reconfigure the encoder with a new (higher max framerate), max fps should
5386 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005387 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005388 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5389 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005390 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005391 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5392
5393 EXPECT_EQ(
5394 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5395 adapted_framerate);
5396
5397 // Trigger underuse, max framerate should go back to codec configured fps.
5398 stats = stats_proxy_->GetStats();
5399 stats.input_frame_rate = adapted_framerate;
5400 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005401 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005402 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5403 EXPECT_EQ(
5404 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5405 kHighFramerate);
5406
5407 video_stream_encoder_->Stop();
5408}
5409
mflodmancc3d4422017-08-03 08:27:51 -07005410TEST_F(VideoStreamEncoderTest,
5411 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005412 const int kFrameWidth = 1280;
5413 const int kFrameHeight = 720;
5414 const int kFramerate = 24;
5415
Henrik Boström381d1092020-05-12 18:49:07 +02005416 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005417 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005418 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005419 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005420 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005421
5422 // Trigger initial configuration.
5423 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005424 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5425 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005426 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 04:21:07 -07005427 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005428 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005429 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005430 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005431
Niels Möller7dc26b72017-12-06 10:27:48 +01005432 EXPECT_EQ(
5433 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5434 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005435
5436 // Trigger overuse, max framerate should be reduced.
5437 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5438 stats.input_frame_rate = kFramerate;
5439 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005440 video_stream_encoder_->TriggerCpuOveruse();
5441 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005442 int adapted_framerate =
5443 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005444 EXPECT_LT(adapted_framerate, kFramerate);
5445
5446 // Change degradation preference to not enable framerate scaling. Target
5447 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005448 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005449 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005450 EXPECT_EQ(
5451 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5452 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005453
mflodmancc3d4422017-08-03 08:27:51 -07005454 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005455}
5456
mflodmancc3d4422017-08-03 08:27:51 -07005457TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005458 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005459 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005460 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5461 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5462 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005463 const int kWidth = 640;
5464 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005465
asaperssonfab67072017-04-04 05:51:49 -07005466 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005467
5468 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005469 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005470
5471 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005472 EXPECT_TRUE_WAIT(
5473 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005474
sprangc5d62e22017-04-02 23:53:04 -07005475 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005476
asaperssonfab67072017-04-04 05:51:49 -07005477 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005478 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005479 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005480
5481 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005482 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005483
Henrik Boström2671dac2020-05-19 16:29:09 +02005484 EXPECT_TRUE_WAIT(
5485 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005486
mflodmancc3d4422017-08-03 08:27:51 -07005487 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005488}
5489
mflodmancc3d4422017-08-03 08:27:51 -07005490TEST_F(VideoStreamEncoderTest,
5491 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005492 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005493 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005494 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5495 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5496 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005497 const int kWidth = 640;
5498 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005499
5500 // We expect the n initial frames to get dropped.
5501 int i;
5502 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005503 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005504 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005505 }
5506 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005507 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005508 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005509
5510 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005511 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005512
mflodmancc3d4422017-08-03 08:27:51 -07005513 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005514}
5515
mflodmancc3d4422017-08-03 08:27:51 -07005516TEST_F(VideoStreamEncoderTest,
5517 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005518 const int kWidth = 640;
5519 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005520 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005521 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005522
5523 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005524 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005525 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005526
asaperssonfab67072017-04-04 05:51:49 -07005527 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005528 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005529 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005530
mflodmancc3d4422017-08-03 08:27:51 -07005531 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005532}
5533
mflodmancc3d4422017-08-03 08:27:51 -07005534TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005535 const int kWidth = 640;
5536 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005537 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005538
5539 VideoEncoderConfig video_encoder_config;
5540 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5541 // Make format different, to force recreation of encoder.
5542 video_encoder_config.video_format.parameters["foo"] = "foo";
5543 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005544 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005545 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005546 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005547
kthelgasonb83797b2017-02-14 11:57:25 -08005548 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005549 video_stream_encoder_->SetSource(&video_source_,
5550 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005551
asaperssonfab67072017-04-04 05:51:49 -07005552 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005553 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005554 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005555
mflodmancc3d4422017-08-03 08:27:51 -07005556 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005557 fake_encoder_.SetQualityScaling(true);
5558}
5559
Åsa Persson139f4dc2019-08-02 09:29:58 +02005560TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005561 webrtc::test::ScopedKeyValueConfig field_trials(
5562 field_trials_,
Åsa Persson139f4dc2019-08-02 09:29:58 +02005563 "WebRTC-Video-QualityScalerSettings/"
5564 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5565 // Reset encoder for field trials to take effect.
5566 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005567 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5568 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 09:29:58 +02005569 const int kWidth = 640;
5570 const int kHeight = 360;
5571
Henrik Boström381d1092020-05-12 18:49:07 +02005572 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005573 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005574 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5575 // Frame should not be dropped.
5576 WaitForEncodedFrame(1);
5577
Henrik Boström381d1092020-05-12 18:49:07 +02005578 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005579 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5580 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5581 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005582 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5583 // Frame should not be dropped.
5584 WaitForEncodedFrame(2);
5585
Henrik Boström381d1092020-05-12 18:49:07 +02005586 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005587 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5588 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5589 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005590 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5591 // Expect to drop this frame, the wait should time out.
5592 ExpectDroppedFrame();
5593
5594 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005595 EXPECT_TRUE_WAIT(
5596 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005597 video_stream_encoder_->Stop();
5598}
5599
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005600TEST_F(VideoStreamEncoderTest,
5601 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005602 webrtc::test::ScopedKeyValueConfig field_trials(
5603 field_trials_,
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005604 "WebRTC-Video-QualityScalerSettings/"
5605 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5606 fake_encoder_.SetQualityScaling(false);
5607 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005608 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5609 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005610 const int kWidth = 640;
5611 const int kHeight = 360;
5612
5613 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005614 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005615 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5616 // Frame should not be dropped.
5617 WaitForEncodedFrame(1);
5618
5619 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5620 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5621 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5622 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5623 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5624 // Frame should not be dropped.
5625 WaitForEncodedFrame(2);
5626
5627 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5628 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5629 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5630 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5631 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5632 // Not dropped since quality scaling is disabled.
5633 WaitForEncodedFrame(3);
5634
5635 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005636 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005637 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5638
5639 video_stream_encoder_->Stop();
5640}
5641
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005642TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005643 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005644 // Set simulcast.
5645 ResetEncoder("VP8", 3, 1, 1, false);
5646 fake_encoder_.SetQualityScaling(true);
5647 const int kWidth = 1280;
5648 const int kHeight = 720;
5649 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005650 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005651 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5652 // Frame should not be dropped.
5653 WaitForEncodedFrame(1);
5654
5655 // Trigger QVGA "singlecast"
5656 // Update the config.
5657 VideoEncoderConfig video_encoder_config;
Jonas Oreland80c87d72022-09-29 15:01:09 +02005658 webrtc::VideoEncoder::EncoderInfo encoder_info;
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005659 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5660 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005661 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005662 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005663 "VP8", /*max qp*/ 56, /*screencast*/ false,
Jonas Oreland80c87d72022-09-29 15:01:09 +02005664 /*screenshare enabled*/ false, encoder_info);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005665 for (auto& layer : video_encoder_config.simulcast_layers) {
5666 layer.num_temporal_layers = 1;
5667 layer.max_framerate = kDefaultFramerate;
5668 }
Asa Persson606d3cb2021-10-04 10:07:11 +02005669 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005670 video_encoder_config.content_type =
5671 VideoEncoderConfig::ContentType::kRealtimeVideo;
5672
5673 video_encoder_config.simulcast_layers[0].active = true;
5674 video_encoder_config.simulcast_layers[1].active = false;
5675 video_encoder_config.simulcast_layers[2].active = false;
5676
5677 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5678 kMaxPayloadLength);
5679 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5680
5681 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5682 // Frame should not be dropped.
5683 WaitForEncodedFrame(2);
5684
5685 // Trigger HD "singlecast"
5686 video_encoder_config.simulcast_layers[0].active = false;
5687 video_encoder_config.simulcast_layers[1].active = false;
5688 video_encoder_config.simulcast_layers[2].active = true;
5689
5690 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5691 kMaxPayloadLength);
5692 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5693
5694 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5695 // Frame should be dropped because of initial frame drop.
5696 ExpectDroppedFrame();
5697
5698 // Expect the sink_wants to specify a scaled frame.
5699 EXPECT_TRUE_WAIT(
5700 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5701 video_stream_encoder_->Stop();
5702}
5703
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005704TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005705 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005706 // Set simulcast.
5707 ResetEncoder("VP9", 1, 1, 3, false);
5708 fake_encoder_.SetQualityScaling(true);
5709 const int kWidth = 1280;
5710 const int kHeight = 720;
5711 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005712 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005713 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5714 // Frame should not be dropped.
5715 WaitForEncodedFrame(1);
5716
5717 // Trigger QVGA "singlecast"
5718 // Update the config.
5719 VideoEncoderConfig video_encoder_config;
5720 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5721 &video_encoder_config);
5722 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5723 vp9_settings.numberOfSpatialLayers = 3;
5724 // Since only one layer is active - automatic resize should be enabled.
5725 vp9_settings.automaticResizeOn = true;
5726 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005727 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005728 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005729 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005730 video_encoder_config.content_type =
5731 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 16:22:11 +02005732 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005733 // which SVC layers are active.
5734 video_encoder_config.simulcast_layers.resize(3);
5735
5736 video_encoder_config.simulcast_layers[0].active = true;
5737 video_encoder_config.simulcast_layers[1].active = false;
5738 video_encoder_config.simulcast_layers[2].active = false;
5739
5740 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5741 kMaxPayloadLength);
5742 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5743
5744 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5745 // Frame should not be dropped.
5746 WaitForEncodedFrame(2);
5747
5748 // Trigger HD "singlecast"
5749 video_encoder_config.simulcast_layers[0].active = false;
5750 video_encoder_config.simulcast_layers[1].active = false;
5751 video_encoder_config.simulcast_layers[2].active = true;
5752
5753 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5754 kMaxPayloadLength);
5755 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5756
5757 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5758 // Frame should be dropped because of initial frame drop.
5759 ExpectDroppedFrame();
5760
5761 // Expect the sink_wants to specify a scaled frame.
5762 EXPECT_TRUE_WAIT(
5763 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5764 video_stream_encoder_->Stop();
5765}
5766
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005767TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005768 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5769 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5770 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5771 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5772 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5773 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5774 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5775 fake_encoder_.SetResolutionBitrateLimits(
5776 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5777
5778 VideoEncoderConfig video_encoder_config;
5779 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5780 &video_encoder_config);
5781 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5782 vp9_settings.numberOfSpatialLayers = 3;
5783 // Since only one layer is active - automatic resize should be enabled.
5784 vp9_settings.automaticResizeOn = true;
5785 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005786 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005787 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005788 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005789 video_encoder_config.content_type =
5790 VideoEncoderConfig::ContentType::kRealtimeVideo;
5791 // Simulcast layers are used to indicate which spatial layers are active.
5792 video_encoder_config.simulcast_layers.resize(3);
5793 video_encoder_config.simulcast_layers[0].active = false;
5794 video_encoder_config.simulcast_layers[1].active = true;
5795 video_encoder_config.simulcast_layers[2].active = false;
5796
5797 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5798 kMaxPayloadLength);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005799
5800 // The encoder bitrate limits for 360p should be used.
5801 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005802 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005803 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5804 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5805 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5806 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5807 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5808 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005809 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005810 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005811 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005812 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005813
5814 // The encoder bitrate limits for 270p should be used.
5815 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02005816 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005817 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5818 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5819 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5820 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5821 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5822 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005823 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005824 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005825 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005826 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005827
5828 video_stream_encoder_->Stop();
5829}
5830
5831TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005832 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5833 VideoEncoderConfig video_encoder_config;
5834 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5835 &video_encoder_config);
5836 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5837 vp9_settings.numberOfSpatialLayers = 3;
5838 // Since only one layer is active - automatic resize should be enabled.
5839 vp9_settings.automaticResizeOn = true;
5840 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005841 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005842 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005843 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005844 video_encoder_config.content_type =
5845 VideoEncoderConfig::ContentType::kRealtimeVideo;
5846 // Simulcast layers are used to indicate which spatial layers are active.
5847 video_encoder_config.simulcast_layers.resize(3);
5848 video_encoder_config.simulcast_layers[0].active = false;
5849 video_encoder_config.simulcast_layers[1].active = true;
5850 video_encoder_config.simulcast_layers[2].active = false;
5851
5852 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5853 kMaxPayloadLength);
Åsa Persson258e9892021-02-25 10:39:51 +01005854
5855 // The default bitrate limits for 360p should be used.
5856 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005857 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5858 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005859 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005860 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005861 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5862 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5863 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5864 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5865 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5866 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005867 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005868 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005869 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005870 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005871
5872 // The default bitrate limits for 270p should be used.
5873 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005874 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5875 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005876 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02005877 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005878 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5879 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5880 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5881 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5882 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5883 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005884 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005885 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005886 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005887 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005888
5889 video_stream_encoder_->Stop();
5890}
5891
5892TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005893 webrtc::test::ScopedKeyValueConfig field_trials(
5894 field_trials_, "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
Åsa Persson258e9892021-02-25 10:39:51 +01005895 VideoEncoderConfig video_encoder_config;
5896 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5897 &video_encoder_config);
5898 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5899 vp9_settings.numberOfSpatialLayers = 3;
5900 // Since only one layer is active - automatic resize should be enabled.
5901 vp9_settings.automaticResizeOn = true;
5902 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005903 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005904 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005905 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005906 video_encoder_config.content_type =
5907 VideoEncoderConfig::ContentType::kRealtimeVideo;
5908 // Simulcast layers are used to indicate which spatial layers are active.
5909 video_encoder_config.simulcast_layers.resize(3);
5910 video_encoder_config.simulcast_layers[0].active = false;
5911 video_encoder_config.simulcast_layers[1].active = true;
5912 video_encoder_config.simulcast_layers[2].active = false;
5913
5914 // Reset encoder for field trials to take effect.
5915 ConfigureEncoder(video_encoder_config.Copy());
5916
5917 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5918 kMaxPayloadLength);
Åsa Persson258e9892021-02-25 10:39:51 +01005919
5920 // The default bitrate limits for 360p should not be used.
5921 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005922 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5923 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005924 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005925 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005926 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5927 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5928 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5929 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5930 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5931 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005932 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005933 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005934
5935 video_stream_encoder_->Stop();
5936}
5937
5938TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5939 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5940 /*num_spatial_layers=*/1, /*screenshare=*/false);
5941
5942 // The default singlecast bitrate limits for 720p should not be used.
5943 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005944 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5945 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005946 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005947 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005948 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5949 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5950 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5951 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5952 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5953 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005954 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005955 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005956
5957 video_stream_encoder_->Stop();
5958}
5959
5960TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005961 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5962 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5963 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5964 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5965 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5966 fake_encoder_.SetResolutionBitrateLimits(
5967 {kEncoderLimits180p, kEncoderLimits720p});
5968
5969 VideoEncoderConfig video_encoder_config;
5970 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5971 &video_encoder_config);
5972 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5973 vp9_settings.numberOfSpatialLayers = 3;
5974 // Since only one layer is active - automatic resize should be enabled.
5975 vp9_settings.automaticResizeOn = true;
5976 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005977 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005978 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005979 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005980 video_encoder_config.content_type =
5981 VideoEncoderConfig::ContentType::kRealtimeVideo;
5982 // Simulcast layers are used to indicate which spatial layers are active.
5983 video_encoder_config.simulcast_layers.resize(3);
5984 video_encoder_config.simulcast_layers[0].active = true;
5985 video_encoder_config.simulcast_layers[1].active = false;
5986 video_encoder_config.simulcast_layers[2].active = false;
5987
5988 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5989 kMaxPayloadLength);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005990
5991 // Limits not applied on lowest stream, limits for 180p should not be used.
5992 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005993 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005994 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5995 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5996 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5997 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5998 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5999 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01006000 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02006001 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01006002 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02006003 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01006004
6005 video_stream_encoder_->Stop();
6006}
6007
6008TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006009 InitialFrameDropActivatesWhenResolutionIncreases) {
6010 const int kWidth = 640;
6011 const int kHeight = 360;
6012
6013 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006014 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006015 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
6016 // Frame should not be dropped.
6017 WaitForEncodedFrame(1);
6018
6019 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006020 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006021 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
6022 // Frame should not be dropped, bitrate not too low for frame.
6023 WaitForEncodedFrame(2);
6024
6025 // Incoming resolution increases.
6026 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
6027 // Expect to drop this frame, bitrate too low for frame.
6028 ExpectDroppedFrame();
6029
6030 // Expect the sink_wants to specify a scaled frame.
6031 EXPECT_TRUE_WAIT(
6032 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
6033 video_stream_encoder_->Stop();
6034}
6035
6036TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
6037 const int kWidth = 640;
6038 const int kHeight = 360;
6039 // So that quality scaling doesn't happen by itself.
6040 fake_encoder_.SetQp(kQpHigh);
6041
6042 AdaptingFrameForwarder source(&time_controller_);
6043 source.set_adaptation_enabled(true);
6044 video_stream_encoder_->SetSource(
6045 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
6046
6047 int timestamp = 1;
6048
6049 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006050 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006051 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6052 WaitForEncodedFrame(timestamp);
6053 timestamp += 9000;
6054 // Long pause to disable all first BWE drop logic.
6055 AdvanceTime(TimeDelta::Millis(1000));
6056
6057 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006058 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006059 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6060 // Not dropped frame, as initial frame drop is disabled by now.
6061 WaitForEncodedFrame(timestamp);
6062 timestamp += 9000;
6063 AdvanceTime(TimeDelta::Millis(100));
6064
6065 // Quality adaptation down.
6066 video_stream_encoder_->TriggerQualityLow();
6067
6068 // Adaptation has an effect.
6069 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6070 5000);
6071
6072 // Frame isn't dropped as initial frame dropper is disabled.
6073 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6074 WaitForEncodedFrame(timestamp);
6075 timestamp += 9000;
6076 AdvanceTime(TimeDelta::Millis(100));
6077
6078 // Quality adaptation up.
6079 video_stream_encoder_->TriggerQualityHigh();
6080
6081 // Adaptation has an effect.
6082 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
6083 5000);
6084
6085 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6086 // Frame should not be dropped, as initial framedropper is off.
6087 WaitForEncodedFrame(timestamp);
6088
6089 video_stream_encoder_->Stop();
6090}
6091
Åsa Persson7f354f82021-02-04 15:52:15 +01006092TEST_F(VideoStreamEncoderTest,
6093 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
6094 const int kMinStartBps360p = 222000;
6095 fake_encoder_.SetResolutionBitrateLimits(
6096 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6097 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6098 800000)});
6099
6100 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6101 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6102 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6103 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
6104 0, 0, 0);
6105 // Frame should not be dropped, bitrate not too low for frame.
6106 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6107 WaitForEncodedFrame(1);
6108
6109 // Incoming resolution increases, initial frame drop activates.
6110 // Frame should be dropped, link allocation too low for frame.
6111 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6112 ExpectDroppedFrame();
6113
6114 // Expect sink_wants to specify a scaled frame.
6115 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6116 5000);
6117 video_stream_encoder_->Stop();
6118}
6119
6120TEST_F(VideoStreamEncoderTest,
6121 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6122 const int kMinStartBps360p = 222000;
6123 fake_encoder_.SetResolutionBitrateLimits(
6124 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6125 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6126 800000)});
6127
6128 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6129 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6130 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6131 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6132 0, 0, 0);
6133 // Frame should not be dropped, bitrate not too low for frame.
6134 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6135 WaitForEncodedFrame(1);
6136
6137 // Incoming resolution increases, initial frame drop activates.
6138 // Frame should be dropped, link allocation not too low for frame.
6139 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6140 WaitForEncodedFrame(2);
6141
6142 video_stream_encoder_->Stop();
6143}
6144
Åsa Perssone644a032019-11-08 15:56:00 +01006145TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006146 webrtc::test::ScopedKeyValueConfig field_trials(
6147 field_trials_,
Åsa Persson06defc42021-09-10 15:28:48 +02006148 "WebRTC-Video-QualityRampupSettings/"
6149 "min_pixels:921600,min_duration_ms:2000/");
6150
6151 const int kWidth = 1280;
6152 const int kHeight = 720;
6153 const int kFps = 10;
6154 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006155
6156 // Reset encoder for field trials to take effect.
6157 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 10:07:11 +02006158 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 11:34:32 +02006159 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01006160 ConfigureEncoder(std::move(config));
6161 fake_encoder_.SetQp(kQpLow);
6162
6163 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006164 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01006165 source.set_adaptation_enabled(true);
6166 video_stream_encoder_->SetSource(&source,
6167 DegradationPreference::MAINTAIN_FRAMERATE);
6168
6169 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 10:07:11 +02006170 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 18:49:07 +02006171 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006172 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006173
6174 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 15:28:48 +02006175 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006176 int64_t timestamp_ms = kFrameIntervalMs;
6177 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6178 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02006179 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6180 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01006181
6182 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02006183 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6184 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006185
Artem Titovab30d722021-07-27 16:22:11 +02006186 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 15:28:48 +02006187 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 15:56:00 +01006188 for (size_t i = 1; i <= 10; i++) {
6189 timestamp_ms += kFrameIntervalMs;
6190 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6191 WaitForEncodedFrame(timestamp_ms);
6192 }
Åsa Persson06defc42021-09-10 15:28:48 +02006193
6194 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6195 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6196 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6197 timestamp_ms += kFrameIntervalMs;
6198 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6199 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 15:56:00 +01006200 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6201 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6202
Åsa Persson06defc42021-09-10 15:28:48 +02006203 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 15:56:00 +01006204 timestamp_ms += kFrameIntervalMs;
6205 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6206 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006207 // The ramp-up code involves the adaptation queue, give it time to execute.
6208 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006209 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006210 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006211
6212 // Frame should not be adapted.
6213 timestamp_ms += kFrameIntervalMs;
6214 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6215 WaitForEncodedFrame(kWidth, kHeight);
6216 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6217
6218 video_stream_encoder_->Stop();
6219}
6220
mflodmancc3d4422017-08-03 08:27:51 -07006221TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006222 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006223 webrtc::test::ScopedKeyValueConfig field_trials(
6224 field_trials_, "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006225 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006226 source.set_adaptation_enabled(true);
6227 video_stream_encoder_->SetSource(&source,
6228 DegradationPreference::MAINTAIN_FRAMERATE);
6229 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006230 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006231 fake_encoder_.SetQp(kQpHigh + 1);
6232 const int kWidth = 1280;
6233 const int kHeight = 720;
6234 const int64_t kFrameIntervalMs = 100;
6235 int64_t timestamp_ms = kFrameIntervalMs;
6236 for (size_t i = 1; i <= 100; i++) {
6237 timestamp_ms += kFrameIntervalMs;
6238 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6239 WaitForEncodedFrame(timestamp_ms);
6240 }
6241 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6242 // for the first time.
6243 // TODO(eshr): We should avoid these waits by using threads with simulated
6244 // time.
6245 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6246 2000 * 2.5 * 2);
6247 timestamp_ms += kFrameIntervalMs;
6248 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6249 WaitForEncodedFrame(timestamp_ms);
6250 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6251 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6252 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6253
6254 // Disable Quality scaling by turning off scaler on the encoder and
6255 // reconfiguring.
6256 fake_encoder_.SetQualityScaling(false);
6257 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6258 kMaxPayloadLength);
6259 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Markus Handell28c71802021-11-08 10:11:55 +01006260 AdvanceTime(TimeDelta::Zero());
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006261 // Since we turned off the quality scaler, the adaptations made by it are
6262 // removed.
6263 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6264 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6265
6266 video_stream_encoder_->Stop();
6267}
6268
6269TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006270 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6271 const int kTooSmallWidth = 10;
6272 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006273 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006274 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006275
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006276 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006277 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006278 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006279 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006280 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006281 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6282
6283 // Trigger adapt down, too small frame, expect no change.
6284 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006285 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006286 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006287 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006288 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6289 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6290
mflodmancc3d4422017-08-03 08:27:51 -07006291 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006292}
6293
mflodmancc3d4422017-08-03 08:27:51 -07006294TEST_F(VideoStreamEncoderTest,
6295 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006296 const int kTooSmallWidth = 10;
6297 const int kTooSmallHeight = 10;
6298 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006299 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006300 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006301
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006302 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006303 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006304 video_stream_encoder_->SetSource(&source,
6305 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006306 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006307 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6308 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6309
6310 // Trigger adapt down, expect limited framerate.
6311 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006312 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006313 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006314 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006315 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6316 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6317 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6318
6319 // Trigger adapt down, too small frame, expect no change.
6320 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006321 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006322 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006323 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006324 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6325 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6326 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6327
mflodmancc3d4422017-08-03 08:27:51 -07006328 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006329}
6330
mflodmancc3d4422017-08-03 08:27:51 -07006331TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006332 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006333 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006334 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006335 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006336 const int kFrameWidth = 1280;
6337 const int kFrameHeight = 720;
6338 video_source_.IncomingCapturedFrame(
6339 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006340 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006341 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006342}
6343
sprangb1ca0732017-02-01 08:38:12 -08006344// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006345TEST_F(VideoStreamEncoderTest,
6346 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006347 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006348 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006349
6350 const int kFrameWidth = 1280;
6351 const int kFrameHeight = 720;
6352 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006353 // requested by
6354 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006355 video_source_.set_adaptation_enabled(true);
6356
6357 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006358 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006359 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006360
6361 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006362 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006363 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006364 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006365 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006366
asaperssonfab67072017-04-04 05:51:49 -07006367 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006368 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006369 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006370 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006371 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006372
mflodmancc3d4422017-08-03 08:27:51 -07006373 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006374}
sprangfe627f32017-03-29 08:24:59 -07006375
mflodmancc3d4422017-08-03 08:27:51 -07006376TEST_F(VideoStreamEncoderTest,
6377 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006378 const int kFrameWidth = 1280;
6379 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006380
Henrik Boström381d1092020-05-12 18:49:07 +02006381 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006382 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006383 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006384 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006385 video_source_.set_adaptation_enabled(true);
6386
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006387 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006388
6389 video_source_.IncomingCapturedFrame(
6390 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006391 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006392
6393 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006394 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006395
6396 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006397 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006398 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006399 video_source_.IncomingCapturedFrame(
6400 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006401 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006402 }
6403
6404 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006405 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006406 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006407 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006408 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006409 video_source_.IncomingCapturedFrame(
6410 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Markus Handell2cfc1af2022-08-19 08:16:48 +00006411 if (!WaitForFrame(kFrameTimeout)) {
sprangc5d62e22017-04-02 23:53:04 -07006412 ++num_frames_dropped;
6413 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006414 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006415 }
6416 }
6417
sprang4847ae62017-06-27 07:06:52 -07006418 // Add some slack to account for frames dropped by the frame dropper.
6419 const int kErrorMargin = 1;
6420 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006421 kErrorMargin);
6422
6423 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006424 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006425 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006426 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006427 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006428 video_source_.IncomingCapturedFrame(
6429 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Markus Handell2cfc1af2022-08-19 08:16:48 +00006430 if (!WaitForFrame(kFrameTimeout)) {
sprangc5d62e22017-04-02 23:53:04 -07006431 ++num_frames_dropped;
6432 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006433 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006434 }
6435 }
sprang4847ae62017-06-27 07:06:52 -07006436 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006437 kErrorMargin);
6438
6439 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006440 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006441 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006442 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006443 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006444 video_source_.IncomingCapturedFrame(
6445 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Markus Handell2cfc1af2022-08-19 08:16:48 +00006446 if (!WaitForFrame(kFrameTimeout)) {
sprangc5d62e22017-04-02 23:53:04 -07006447 ++num_frames_dropped;
6448 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006449 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006450 }
6451 }
sprang4847ae62017-06-27 07:06:52 -07006452 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006453 kErrorMargin);
6454
6455 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006456 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006457 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006458 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006459 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006460 video_source_.IncomingCapturedFrame(
6461 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Markus Handell2cfc1af2022-08-19 08:16:48 +00006462 if (!WaitForFrame(kFrameTimeout)) {
sprangc5d62e22017-04-02 23:53:04 -07006463 ++num_frames_dropped;
6464 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006465 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006466 }
6467 }
6468 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6469
mflodmancc3d4422017-08-03 08:27:51 -07006470 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006471}
6472
mflodmancc3d4422017-08-03 08:27:51 -07006473TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006474 const int kFramerateFps = 5;
6475 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006476 const int kFrameWidth = 1280;
6477 const int kFrameHeight = 720;
6478
sprang4847ae62017-06-27 07:06:52 -07006479 // Reconfigure encoder with two temporal layers and screensharing, which will
6480 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006481 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006482
Henrik Boström381d1092020-05-12 18:49:07 +02006483 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006484 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006485 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006486 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006487 video_source_.set_adaptation_enabled(true);
6488
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006489 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006490
6491 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006492 rtc::VideoSinkWants last_wants;
6493 do {
6494 last_wants = video_source_.sink_wants();
6495
sprangc5d62e22017-04-02 23:53:04 -07006496 // Insert frames to get a new fps estimate...
6497 for (int j = 0; j < kFramerateFps; ++j) {
6498 video_source_.IncomingCapturedFrame(
6499 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006500 if (video_source_.last_sent_width()) {
6501 sink_.WaitForEncodedFrame(timestamp_ms);
6502 }
sprangc5d62e22017-04-02 23:53:04 -07006503 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006504 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006505 }
6506 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006507 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006508 } while (video_source_.sink_wants().max_framerate_fps <
6509 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006510
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006511 EXPECT_THAT(video_source_.sink_wants(),
6512 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006513
mflodmancc3d4422017-08-03 08:27:51 -07006514 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006515}
asaperssonf7e294d2017-06-13 23:25:22 -07006516
mflodmancc3d4422017-08-03 08:27:51 -07006517TEST_F(VideoStreamEncoderTest,
6518 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006519 const int kWidth = 1280;
6520 const int kHeight = 720;
6521 const int64_t kFrameIntervalMs = 150;
6522 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006523 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006524 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006525
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006526 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006527 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006528 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006529 video_stream_encoder_->SetSource(&source,
6530 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006531 timestamp_ms += kFrameIntervalMs;
6532 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006533 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006534 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006535 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6536 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6537 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6538
6539 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006540 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006541 timestamp_ms += kFrameIntervalMs;
6542 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006543 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006544 EXPECT_THAT(source.sink_wants(),
6545 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006546 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6547 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6548 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6549
6550 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006551 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006552 timestamp_ms += kFrameIntervalMs;
6553 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006554 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006555 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006556 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6557 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6558 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6559
6560 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006561 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006562 timestamp_ms += kFrameIntervalMs;
6563 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006564 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006565 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006566 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6567 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6568 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6569
6570 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006571 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006572 timestamp_ms += kFrameIntervalMs;
6573 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006574 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006575 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006576 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6577 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6578 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6579
6580 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006581 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006582 timestamp_ms += kFrameIntervalMs;
6583 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006584 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006585 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006586 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6587 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6588 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6589
6590 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006591 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006592 timestamp_ms += kFrameIntervalMs;
6593 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006594 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006595 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006596 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6597 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6598 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6599
6600 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006601 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006602 timestamp_ms += kFrameIntervalMs;
6603 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006604 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006605 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006606 rtc::VideoSinkWants last_wants = source.sink_wants();
6607 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6608 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6609 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6610
6611 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006612 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006613 timestamp_ms += kFrameIntervalMs;
6614 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006615 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006616 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006617 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6618 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6619 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6620
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006621 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006622 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006623 timestamp_ms += kFrameIntervalMs;
6624 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006625 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006626 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006627 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6628 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6629 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6630
6631 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006632 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006633 timestamp_ms += kFrameIntervalMs;
6634 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006635 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006636 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006637 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6638 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6639 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6640
6641 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006642 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006643 timestamp_ms += kFrameIntervalMs;
6644 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006645 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006646 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006647 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6648 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6649 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6650
6651 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006652 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006653 timestamp_ms += kFrameIntervalMs;
6654 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006655 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006656 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006657 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6658 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6659 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6660
6661 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006662 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006663 timestamp_ms += kFrameIntervalMs;
6664 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006665 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006666 EXPECT_THAT(source.sink_wants(), FpsMax());
6667 EXPECT_EQ(source.sink_wants().max_pixel_count,
6668 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006669 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6670 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6671 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6672
6673 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006674 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006675 timestamp_ms += kFrameIntervalMs;
6676 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006677 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006678 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006679 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6680 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6681 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6682
Åsa Persson30ab0152019-08-27 12:22:33 +02006683 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006684 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006685 timestamp_ms += kFrameIntervalMs;
6686 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006687 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006688 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006689 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006690 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6691 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6692 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6693
6694 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006695 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006696 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006697 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6698
mflodmancc3d4422017-08-03 08:27:51 -07006699 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006700}
6701
mflodmancc3d4422017-08-03 08:27:51 -07006702TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006703 const int kWidth = 1280;
6704 const int kHeight = 720;
6705 const int64_t kFrameIntervalMs = 150;
6706 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006707 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006708 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006709
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006710 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006711 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006712 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006713 video_stream_encoder_->SetSource(&source,
6714 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006715 timestamp_ms += kFrameIntervalMs;
6716 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006717 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006718 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006719 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6720 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6721 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6722 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6723 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6724 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6725
6726 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006727 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006728 timestamp_ms += kFrameIntervalMs;
6729 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006730 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006731 EXPECT_THAT(source.sink_wants(),
6732 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006733 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6734 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6735 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6736 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6737 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6738 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6739
6740 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006741 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006742 timestamp_ms += kFrameIntervalMs;
6743 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006744 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006745 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006746 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6747 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6748 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6749 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6750 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6751 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6752
6753 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006754 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006755 timestamp_ms += kFrameIntervalMs;
6756 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006757 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006758 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006759 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006760 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6761 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6762 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6763 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6764 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6765
Evan Shrubsole64469032020-06-11 10:45:29 +02006766 // Trigger cpu adapt up, expect no change since QP is most limited.
6767 {
6768 // Store current sink wants since we expect no change and if there is no
6769 // change then last_wants() is not updated.
6770 auto previous_sink_wants = source.sink_wants();
6771 video_stream_encoder_->TriggerCpuUnderuse();
6772 timestamp_ms += kFrameIntervalMs;
6773 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6774 WaitForEncodedFrame(timestamp_ms);
6775 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6776 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6777 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6778 }
6779
6780 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6781 video_stream_encoder_->TriggerQualityHigh();
6782 timestamp_ms += kFrameIntervalMs;
6783 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6784 WaitForEncodedFrame(timestamp_ms);
6785 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6786 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6787 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6788 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6789 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6790 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6791 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6792
6793 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6794 // expect increased resolution (960x540@30fps).
6795 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006796 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006797 timestamp_ms += kFrameIntervalMs;
6798 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006799 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006800 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006801 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6802 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6803 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6804 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6805 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006806 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006807
Evan Shrubsole64469032020-06-11 10:45:29 +02006808 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6809 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006810 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006811 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006812 timestamp_ms += kFrameIntervalMs;
6813 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006814 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006815 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006816 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006817 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6818 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6819 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6820 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6821 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006822 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006823
6824 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006825 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006826 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006827 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006828 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006829
mflodmancc3d4422017-08-03 08:27:51 -07006830 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006831}
6832
mflodmancc3d4422017-08-03 08:27:51 -07006833TEST_F(VideoStreamEncoderTest,
6834 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006835 const int kWidth = 640;
6836 const int kHeight = 360;
6837 const int kFpsLimit = 15;
6838 const int64_t kFrameIntervalMs = 150;
6839 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006840 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006841 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006842
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006843 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006844 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006845 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006846 video_stream_encoder_->SetSource(&source,
6847 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006848 timestamp_ms += kFrameIntervalMs;
6849 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006850 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006851 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006852 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6853 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6854 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6855 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6856 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6857 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6858
6859 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006860 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006861 timestamp_ms += kFrameIntervalMs;
6862 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006863 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006864 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006865 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6866 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6867 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6868 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6869 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6870 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6871
6872 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006873 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006874 timestamp_ms += kFrameIntervalMs;
6875 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006876 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006877 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006878 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006879 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006880 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6881 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6882 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6883 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6884
Evan Shrubsole64469032020-06-11 10:45:29 +02006885 // Trigger cpu adapt up, expect no change because quality is most limited.
6886 {
6887 auto previous_sink_wants = source.sink_wants();
6888 // Store current sink wants since we expect no change ind if there is no
6889 // change then last__wants() is not updated.
6890 video_stream_encoder_->TriggerCpuUnderuse();
6891 timestamp_ms += kFrameIntervalMs;
6892 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6893 WaitForEncodedFrame(timestamp_ms);
6894 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6895 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6896 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6897 }
6898
6899 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6900 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006901 timestamp_ms += kFrameIntervalMs;
6902 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006903 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006904 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006905 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6906 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6907 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006908 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6909 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6910 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006911
Evan Shrubsole64469032020-06-11 10:45:29 +02006912 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006913 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006914 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006915 timestamp_ms += kFrameIntervalMs;
6916 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006917 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006918 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006919 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6920 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6921 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6922 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6923 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006924 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006925
6926 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006927 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006928 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006929 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006930 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006931
mflodmancc3d4422017-08-03 08:27:51 -07006932 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006933}
6934
mflodmancc3d4422017-08-03 08:27:51 -07006935TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006936 const int kFrameWidth = 1920;
6937 const int kFrameHeight = 1080;
Ying Wangdb0d5862022-04-28 19:26:22 +02006938 // 2/3 of 1920.
6939 const int kAdaptedFrameWidth = 1280;
6940 // 2/3 of 1080.
6941 const int kAdaptedFrameHeight = 720;
ilnik6b826ef2017-06-16 06:53:48 -07006942 const int kFramerate = 24;
6943
Henrik Boström381d1092020-05-12 18:49:07 +02006944 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006945 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006946 // Trigger reconfigure encoder (without resetting the entire instance).
6947 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006948 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6949 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02006950 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 06:53:48 -07006951 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006952 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006953 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006954 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006955 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006956
6957 video_source_.set_adaptation_enabled(true);
6958
6959 video_source_.IncomingCapturedFrame(
6960 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006961 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006962
6963 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006964 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006965 video_source_.IncomingCapturedFrame(
6966 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006967 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006968
mflodmancc3d4422017-08-03 08:27:51 -07006969 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006970}
6971
mflodmancc3d4422017-08-03 08:27:51 -07006972TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006973 const int kFrameWidth = 1280;
6974 const int kFrameHeight = 720;
6975 const int kLowFps = 2;
6976 const int kHighFps = 30;
6977
Henrik Boström381d1092020-05-12 18:49:07 +02006978 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006979 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006980
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006981 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006982 max_framerate_ = kLowFps;
6983
6984 // Insert 2 seconds of 2fps video.
6985 for (int i = 0; i < kLowFps * 2; ++i) {
6986 video_source_.IncomingCapturedFrame(
6987 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6988 WaitForEncodedFrame(timestamp_ms);
6989 timestamp_ms += 1000 / kLowFps;
6990 }
6991
6992 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006993 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006994 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006995 video_source_.IncomingCapturedFrame(
6996 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6997 WaitForEncodedFrame(timestamp_ms);
6998 timestamp_ms += 1000 / kLowFps;
6999
7000 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
7001
7002 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02007003 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
Markus Handell2cfc1af2022-08-19 08:16:48 +00007004 constexpr TimeDelta kFrameInterval = TimeDelta::Seconds(1) / kHighFps;
sprang4847ae62017-06-27 07:06:52 -07007005 max_framerate_ = kHighFps;
7006 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
7007 video_source_.IncomingCapturedFrame(
7008 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7009 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
7010 // be dropped if the encoder hans't been updated with the new higher target
7011 // framerate yet, causing it to overshoot the target bitrate and then
7012 // suffering the wrath of the media optimizer.
Markus Handell2cfc1af2022-08-19 08:16:48 +00007013 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameInterval);
7014 timestamp_ms += kFrameInterval.ms();
sprang4847ae62017-06-27 07:06:52 -07007015 }
7016
7017 // Don expect correct measurement just yet, but it should be higher than
7018 // before.
7019 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
7020
mflodmancc3d4422017-08-03 08:27:51 -07007021 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007022}
7023
mflodmancc3d4422017-08-03 08:27:51 -07007024TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07007025 const int kFrameWidth = 1280;
7026 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 15:09:05 +02007027 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01007028 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02007029 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07007030
Henrik Boström381d1092020-05-12 18:49:07 +02007031 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007032 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07007033 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07007034
7035 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007036 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07007037 video_source_.IncomingCapturedFrame(
7038 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7039 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02007040 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007041
7042 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02007043 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007044 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07007045
7046 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02007047 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007048 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07007049
Per Kjellanderdcef6412020-10-07 15:09:05 +02007050 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07007051 video_source_.IncomingCapturedFrame(
7052 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7053 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02007054 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007055
mflodmancc3d4422017-08-03 08:27:51 -07007056 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007057}
ilnik6b826ef2017-06-16 06:53:48 -07007058
Niels Möller4db138e2018-04-19 09:04:13 +02007059TEST_F(VideoStreamEncoderTest,
7060 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
7061 const int kFrameWidth = 1280;
7062 const int kFrameHeight = 720;
Markus Handell8e4197b2022-05-30 15:45:28 +02007063 const test::ScopedKeyValueConfig kFieldTrials;
7064 const CpuOveruseOptions default_options(kFieldTrials);
Henrik Boström381d1092020-05-12 18:49:07 +02007065 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007066 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007067 video_source_.IncomingCapturedFrame(
7068 CreateFrame(1, kFrameWidth, kFrameHeight));
7069 WaitForEncodedFrame(1);
7070 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7071 .low_encode_usage_threshold_percent,
7072 default_options.low_encode_usage_threshold_percent);
7073 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7074 .high_encode_usage_threshold_percent,
7075 default_options.high_encode_usage_threshold_percent);
7076 video_stream_encoder_->Stop();
7077}
7078
7079TEST_F(VideoStreamEncoderTest,
7080 HigherCpuAdaptationThresholdsForHardwareEncoder) {
7081 const int kFrameWidth = 1280;
7082 const int kFrameHeight = 720;
Markus Handell8e4197b2022-05-30 15:45:28 +02007083 const test::ScopedKeyValueConfig kFieldTrials;
7084 CpuOveruseOptions hardware_options(kFieldTrials);
Niels Möller4db138e2018-04-19 09:04:13 +02007085 hardware_options.low_encode_usage_threshold_percent = 150;
7086 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01007087 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02007088
Henrik Boström381d1092020-05-12 18:49:07 +02007089 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007090 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007091 video_source_.IncomingCapturedFrame(
7092 CreateFrame(1, kFrameWidth, kFrameHeight));
7093 WaitForEncodedFrame(1);
7094 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7095 .low_encode_usage_threshold_percent,
7096 hardware_options.low_encode_usage_threshold_percent);
7097 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7098 .high_encode_usage_threshold_percent,
7099 hardware_options.high_encode_usage_threshold_percent);
7100 video_stream_encoder_->Stop();
7101}
7102
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007103TEST_F(VideoStreamEncoderTest,
7104 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
7105 const int kFrameWidth = 1280;
7106 const int kFrameHeight = 720;
7107
Markus Handell8e4197b2022-05-30 15:45:28 +02007108 const test::ScopedKeyValueConfig kFieldTrials;
7109 const CpuOveruseOptions default_options(kFieldTrials);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007110 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007111 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007112 video_source_.IncomingCapturedFrame(
7113 CreateFrame(1, kFrameWidth, kFrameHeight));
7114 WaitForEncodedFrame(1);
7115 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7116 .low_encode_usage_threshold_percent,
7117 default_options.low_encode_usage_threshold_percent);
7118 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7119 .high_encode_usage_threshold_percent,
7120 default_options.high_encode_usage_threshold_percent);
7121
Markus Handell8e4197b2022-05-30 15:45:28 +02007122 CpuOveruseOptions hardware_options(kFieldTrials);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007123 hardware_options.low_encode_usage_threshold_percent = 150;
7124 hardware_options.high_encode_usage_threshold_percent = 200;
7125 fake_encoder_.SetIsHardwareAccelerated(true);
7126
7127 video_source_.IncomingCapturedFrame(
7128 CreateFrame(2, kFrameWidth, kFrameHeight));
7129 WaitForEncodedFrame(2);
7130
7131 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7132 .low_encode_usage_threshold_percent,
7133 hardware_options.low_encode_usage_threshold_percent);
7134 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7135 .high_encode_usage_threshold_percent,
7136 hardware_options.high_encode_usage_threshold_percent);
7137
7138 video_stream_encoder_->Stop();
7139}
7140
Niels Möller6bb5ab92019-01-11 11:11:10 +01007141TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7142 const int kFrameWidth = 320;
7143 const int kFrameHeight = 240;
7144 const int kFps = 30;
Asa Persson606d3cb2021-10-04 10:07:11 +02007145 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007146 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7147
Henrik Boström381d1092020-05-12 18:49:07 +02007148 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007149 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007150
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007151 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007152 max_framerate_ = kFps;
7153
7154 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7155 fake_encoder_.SimulateOvershoot(1.0);
7156 int num_dropped = 0;
7157 for (int i = 0; i < kNumFramesInRun; ++i) {
7158 video_source_.IncomingCapturedFrame(
7159 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7160 // Wait up to two frame durations for a frame to arrive.
Markus Handell2cfc1af2022-08-19 08:16:48 +00007161 if (!TimedWaitForEncodedFrame(timestamp_ms,
7162 2 * TimeDelta::Seconds(1) / kFps)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01007163 ++num_dropped;
7164 }
7165 timestamp_ms += 1000 / kFps;
7166 }
7167
Erik Språnga8d48ab2019-02-08 14:17:40 +01007168 // Framerate should be measured to be near the expected target rate.
7169 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7170
7171 // Frame drops should be within 5% of expected 0%.
7172 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007173
7174 // Make encoder produce frames at double the expected bitrate during 3 seconds
7175 // of video, verify number of drops. Rate needs to be slightly changed in
7176 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01007177 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02007178 const RateControlSettings trials =
7179 RateControlSettings::ParseFromFieldTrials();
7180 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01007181 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02007182 // frame dropping since the adjuter will try to just lower the target
7183 // bitrate rather than drop frames. If network headroom can be used, it
7184 // doesn't push back as hard so we don't need quite as much overshoot.
7185 // These numbers are unfortunately a bit magical but there's not trivial
7186 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01007187 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01007188 }
7189 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02007190 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007191 kTargetBitrate + DataRate::KilobitsPerSec(1),
7192 kTargetBitrate + DataRate::KilobitsPerSec(1),
7193 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007194 num_dropped = 0;
7195 for (int i = 0; i < kNumFramesInRun; ++i) {
7196 video_source_.IncomingCapturedFrame(
7197 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7198 // Wait up to two frame durations for a frame to arrive.
Markus Handell2cfc1af2022-08-19 08:16:48 +00007199 if (!TimedWaitForEncodedFrame(timestamp_ms,
7200 2 * TimeDelta::Seconds(1) / kFps)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01007201 ++num_dropped;
7202 }
7203 timestamp_ms += 1000 / kFps;
7204 }
7205
Henrik Boström381d1092020-05-12 18:49:07 +02007206 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007207 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007208
7209 // Target framerate should be still be near the expected target, despite
7210 // the frame drops.
7211 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7212
7213 // Frame drops should be within 5% of expected 50%.
7214 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007215
7216 video_stream_encoder_->Stop();
7217}
7218
7219TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7220 const int kFrameWidth = 320;
7221 const int kFrameHeight = 240;
7222 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 10:07:11 +02007223 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007224
7225 ASSERT_GT(max_framerate_, kActualInputFps);
7226
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007227 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007228 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007229 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007230 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007231
7232 // Insert 3 seconds of video, with an input fps lower than configured max.
7233 for (int i = 0; i < kActualInputFps * 3; ++i) {
7234 video_source_.IncomingCapturedFrame(
7235 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7236 // Wait up to two frame durations for a frame to arrive.
7237 WaitForEncodedFrame(timestamp_ms);
7238 timestamp_ms += 1000 / kActualInputFps;
7239 }
7240
7241 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7242
7243 video_stream_encoder_->Stop();
7244}
7245
Markus Handell9a478b52021-11-18 16:07:01 +01007246TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007247 VideoFrame::UpdateRect rect;
Markus Handell9a478b52021-11-18 16:07:01 +01007248 test::FrameForwarder source;
7249 video_stream_encoder_->SetSource(&source,
7250 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02007251 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007252 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007253
Markus Handell9a478b52021-11-18 16:07:01 +01007254 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(1, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007255 WaitForEncodedFrame(1);
7256 // On the very first frame full update should be forced.
7257 rect = fake_encoder_.GetLastUpdateRect();
7258 EXPECT_EQ(rect.offset_x, 0);
7259 EXPECT_EQ(rect.offset_y, 0);
7260 EXPECT_EQ(rect.height, codec_height_);
7261 EXPECT_EQ(rect.width, codec_width_);
Markus Handell9a478b52021-11-18 16:07:01 +01007262 // Frame with NTP timestamp 2 will be dropped due to outstanding frames
7263 // scheduled for processing during encoder queue processing of frame 2.
7264 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(2, nullptr, 1));
7265 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(3, nullptr, 10));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007266 WaitForEncodedFrame(3);
7267 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7268 rect = fake_encoder_.GetLastUpdateRect();
7269 EXPECT_EQ(rect.offset_x, 1);
7270 EXPECT_EQ(rect.offset_y, 0);
7271 EXPECT_EQ(rect.width, 10);
7272 EXPECT_EQ(rect.height, 1);
7273
Markus Handell9a478b52021-11-18 16:07:01 +01007274 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(4, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007275 WaitForEncodedFrame(4);
7276 // Previous frame was encoded, so no accumulation should happen.
7277 rect = fake_encoder_.GetLastUpdateRect();
7278 EXPECT_EQ(rect.offset_x, 0);
7279 EXPECT_EQ(rect.offset_y, 0);
7280 EXPECT_EQ(rect.width, 1);
7281 EXPECT_EQ(rect.height, 1);
7282
7283 video_stream_encoder_->Stop();
7284}
7285
Erik Språngd7329ca2019-02-21 21:19:53 +01007286TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007287 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007288 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007289
7290 // First frame is always keyframe.
7291 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7292 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007293 EXPECT_THAT(
7294 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007295 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007296
7297 // Insert delta frame.
7298 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7299 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007300 EXPECT_THAT(
7301 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007302 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007303
7304 // Request next frame be a key-frame.
7305 video_stream_encoder_->SendKeyFrame();
7306 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7307 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007308 EXPECT_THAT(
7309 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007310 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007311
7312 video_stream_encoder_->Stop();
7313}
7314
7315TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7316 // Setup simulcast with three streams.
7317 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007318 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007319 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7320 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007321 // Wait for all three layers before triggering event.
7322 sink_.SetNumExpectedLayers(3);
7323
7324 // First frame is always keyframe.
7325 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7326 WaitForEncodedFrame(1);
7327 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007328 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7329 VideoFrameType::kVideoFrameKey,
7330 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007331
7332 // Insert delta frame.
7333 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7334 WaitForEncodedFrame(2);
7335 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007336 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7337 VideoFrameType::kVideoFrameDelta,
7338 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007339
7340 // Request next frame be a key-frame.
7341 // Only first stream is configured to produce key-frame.
7342 video_stream_encoder_->SendKeyFrame();
7343 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7344 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007345
7346 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7347 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007348 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007349 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007350 VideoFrameType::kVideoFrameKey,
7351 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007352
7353 video_stream_encoder_->Stop();
7354}
7355
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007356TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007357 // SPS contains VUI with restrictions on the maximum number of reordered
7358 // pictures, there is no need to rewrite the bitstream to enable faster
7359 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007360 ResetEncoder("H264", 1, 1, 1, false);
7361
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007362 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007363 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007364 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007365
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007366 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 10:07:11 +02007367 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007368
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007369 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7370 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007371
7372 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007373 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007374
7375 video_stream_encoder_->Stop();
7376}
7377
7378TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007379 // SPS does not contain VUI, the bitstream is will be rewritten with added
7380 // VUI with restrictions on the maximum number of reordered pictures to
7381 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007382 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7383 0x00, 0x00, 0x03, 0x03, 0xF4,
7384 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007385 ResetEncoder("H264", 1, 1, 1, false);
7386
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007387 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007388 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007389 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007390
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007391 fake_encoder_.SetEncodedImageData(
7392 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007393
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007394 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7395 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007396
7397 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007398 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007399
7400 video_stream_encoder_->Stop();
7401}
7402
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007403TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7404 const int kFrameWidth = 1280;
7405 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 10:07:11 +02007406 const DataRate kTargetBitrate =
7407 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007408
Henrik Boström381d1092020-05-12 18:49:07 +02007409 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007410 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007411 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7412
7413 // Insert a first video frame. It should be dropped because of downscale in
7414 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007415 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007416 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7417 frame.set_rotation(kVideoRotation_270);
7418 video_source_.IncomingCapturedFrame(frame);
7419
7420 ExpectDroppedFrame();
7421
7422 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007423 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007424 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7425 frame.set_rotation(kVideoRotation_90);
7426 video_source_.IncomingCapturedFrame(frame);
7427
7428 WaitForEncodedFrame(timestamp_ms);
7429 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7430
7431 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007432 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007433 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7434 frame.set_rotation(kVideoRotation_180);
7435 video_source_.IncomingCapturedFrame(frame);
7436
7437 WaitForEncodedFrame(timestamp_ms);
7438 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7439
7440 video_stream_encoder_->Stop();
7441}
7442
Erik Språng5056af02019-09-02 15:53:11 +02007443TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7444 const int kFrameWidth = 320;
7445 const int kFrameHeight = 180;
7446
7447 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007448 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007449 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7450 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7451 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007452 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007453 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007454 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007455
7456 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007457 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007458 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7459 frame.set_rotation(kVideoRotation_270);
7460 video_source_.IncomingCapturedFrame(frame);
7461 WaitForEncodedFrame(timestamp_ms);
7462
7463 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 10:07:11 +02007464 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007465 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7466 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007467 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007468 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007469 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007470 /*link_allocation=*/target_rate,
7471 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007472 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007473 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007474 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7475
7476 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7477 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7478 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007479 DataRate allocation_sum =
7480 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007481 EXPECT_EQ(min_rate, allocation_sum);
7482 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7483
7484 video_stream_encoder_->Stop();
7485}
7486
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007487TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007488 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007489 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007490 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007491 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007492 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7493 WaitForEncodedFrame(1);
7494
7495 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7496 ASSERT_TRUE(prev_rate_settings.has_value());
7497 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7498 kDefaultFramerate);
7499
7500 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7501 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7502 timestamp_ms += 1000 / kDefaultFramerate;
7503 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7504 WaitForEncodedFrame(timestamp_ms);
7505 }
7506 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7507 kDefaultFramerate);
7508 // Capture larger frame to trigger a reconfigure.
7509 codec_height_ *= 2;
7510 codec_width_ *= 2;
7511 timestamp_ms += 1000 / kDefaultFramerate;
7512 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7513 WaitForEncodedFrame(timestamp_ms);
7514
7515 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7516 auto current_rate_settings =
7517 fake_encoder_.GetAndResetLastRateControlSettings();
7518 // Ensure we have actually reconfigured twice
7519 // The rate settings should have been set again even though
7520 // they haven't changed.
7521 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007522 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007523
7524 video_stream_encoder_->Stop();
7525}
7526
philipeld9cc8c02019-09-16 14:53:40 +02007527struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007528 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007529 MOCK_METHOD(void,
7530 RequestEncoderSwitch,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007531 (const webrtc::SdpVideoFormat& format,
7532 bool allow_default_fallback),
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007533 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007534};
7535
philipel9b058032020-02-10 11:30:00 +01007536TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7537 constexpr int kDontCare = 100;
7538 StrictMock<MockEncoderSelector> encoder_selector;
7539 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7540 &fake_encoder_, &encoder_selector);
7541 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7542
7543 // Reset encoder for new configuration to take effect.
7544 ConfigureEncoder(video_encoder_config_.Copy());
7545
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007546 EXPECT_CALL(encoder_selector, OnCurrentEncoder);
philipel9b058032020-02-10 11:30:00 +01007547
7548 video_source_.IncomingCapturedFrame(
7549 CreateFrame(kDontCare, kDontCare, kDontCare));
Markus Handell28c71802021-11-08 10:11:55 +01007550 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007551 video_stream_encoder_->Stop();
7552
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007553 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007554 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007555 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7556 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007557 video_stream_encoder_.reset();
7558}
7559
7560TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7561 constexpr int kDontCare = 100;
7562
7563 NiceMock<MockEncoderSelector> encoder_selector;
7564 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7565 video_send_config_.encoder_settings.encoder_switch_request_callback =
7566 &switch_callback;
7567 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7568 &fake_encoder_, &encoder_selector);
7569 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7570
7571 // Reset encoder for new configuration to take effect.
7572 ConfigureEncoder(video_encoder_config_.Copy());
7573
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007574 ON_CALL(encoder_selector, OnAvailableBitrate)
philipel9b058032020-02-10 11:30:00 +01007575 .WillByDefault(Return(SdpVideoFormat("AV1")));
7576 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007577 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7578 /*allow_default_fallback=*/false));
philipel9b058032020-02-10 11:30:00 +01007579
Henrik Boström381d1092020-05-12 18:49:07 +02007580 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007581 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7582 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7583 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007584 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007585 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007586 /*cwnd_reduce_ratio=*/0);
Markus Handell28c71802021-11-08 10:11:55 +01007587 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007588
7589 video_stream_encoder_->Stop();
7590}
7591
philipel6daa3042022-04-11 10:48:28 +02007592TEST_F(VideoStreamEncoderTest, EncoderSelectorResolutionSwitch) {
7593 NiceMock<MockEncoderSelector> encoder_selector;
7594 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7595 video_send_config_.encoder_settings.encoder_switch_request_callback =
7596 &switch_callback;
7597 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7598 &fake_encoder_, &encoder_selector);
7599 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7600
7601 // Reset encoder for new configuration to take effect.
7602 ConfigureEncoder(video_encoder_config_.Copy());
7603
7604 EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(640, 480)))
7605 .WillOnce(Return(absl::nullopt));
7606 EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(320, 240)))
7607 .WillOnce(Return(SdpVideoFormat("AV1")));
7608 EXPECT_CALL(switch_callback,
7609 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7610 /*allow_default_fallback=*/false));
7611
7612 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7613 /*target_bitrate=*/DataRate::KilobitsPerSec(800),
7614 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(1000),
7615 /*link_allocation=*/DataRate::KilobitsPerSec(1000),
7616 /*fraction_lost=*/0,
7617 /*round_trip_time_ms=*/0,
7618 /*cwnd_reduce_ratio=*/0);
7619
7620 video_source_.IncomingCapturedFrame(CreateFrame(1, 640, 480));
7621 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 480));
7622 video_source_.IncomingCapturedFrame(CreateFrame(3, 320, 240));
7623
7624 AdvanceTime(TimeDelta::Zero());
7625
7626 video_stream_encoder_->Stop();
7627}
7628
philipel9b058032020-02-10 11:30:00 +01007629TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7630 constexpr int kSufficientBitrateToNotDrop = 1000;
7631 constexpr int kDontCare = 100;
7632
7633 NiceMock<MockVideoEncoder> video_encoder;
7634 NiceMock<MockEncoderSelector> encoder_selector;
7635 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7636 video_send_config_.encoder_settings.encoder_switch_request_callback =
7637 &switch_callback;
7638 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7639 &video_encoder, &encoder_selector);
7640 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7641
7642 // Reset encoder for new configuration to take effect.
7643 ConfigureEncoder(video_encoder_config_.Copy());
7644
7645 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7646 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7647 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007648 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007649 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7650 /*stable_target_bitrate=*/
7651 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7652 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007653 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007654 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007655 /*cwnd_reduce_ratio=*/0);
7656
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007657 ON_CALL(video_encoder, Encode)
philipel9b058032020-02-10 11:30:00 +01007658 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007659 ON_CALL(encoder_selector, OnEncoderBroken)
philipel9b058032020-02-10 11:30:00 +01007660 .WillByDefault(Return(SdpVideoFormat("AV2")));
7661
7662 rtc::Event encode_attempted;
7663 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007664 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7665 /*allow_default_fallback=*/true))
7666 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
philipel9b058032020-02-10 11:30:00 +01007667
7668 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
Markus Handell2cfc1af2022-08-19 08:16:48 +00007669 encode_attempted.Wait(TimeDelta::Seconds(3));
philipel9b058032020-02-10 11:30:00 +01007670
Markus Handell28c71802021-11-08 10:11:55 +01007671 AdvanceTime(TimeDelta::Zero());
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007672
philipel9b058032020-02-10 11:30:00 +01007673 video_stream_encoder_->Stop();
7674
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007675 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7676 // to it's factory, so in order for the encoder instance in the
7677 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7678 // reset the `video_stream_encoder_` here.
7679 video_stream_encoder_.reset();
7680}
7681
7682TEST_F(VideoStreamEncoderTest, SwitchEncoderOnInitFailureWithEncoderSelector) {
7683 NiceMock<MockVideoEncoder> video_encoder;
7684 NiceMock<MockEncoderSelector> encoder_selector;
7685 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7686 video_send_config_.encoder_settings.encoder_switch_request_callback =
7687 &switch_callback;
7688 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7689 &video_encoder, &encoder_selector);
7690 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7691
7692 // Reset encoder for new configuration to take effect.
7693 ConfigureEncoder(video_encoder_config_.Copy());
7694
7695 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7696 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7697 /*round_trip_time_ms=*/0,
7698 /*cwnd_reduce_ratio=*/0);
7699 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7700
7701 ON_CALL(video_encoder, InitEncode(_, _))
7702 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7703 ON_CALL(encoder_selector, OnEncoderBroken)
7704 .WillByDefault(Return(SdpVideoFormat("AV2")));
7705
7706 rtc::Event encode_attempted;
7707 EXPECT_CALL(switch_callback,
7708 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7709 /*allow_default_fallback=*/true))
7710 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7711
7712 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
Markus Handell2cfc1af2022-08-19 08:16:48 +00007713 encode_attempted.Wait(TimeDelta::Seconds(3));
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007714
7715 AdvanceTime(TimeDelta::Zero());
7716
7717 video_stream_encoder_->Stop();
7718
7719 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7720 // to it's factory, so in order for the encoder instance in the
7721 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7722 // reset the `video_stream_encoder_` here.
7723 video_stream_encoder_.reset();
7724}
7725
7726TEST_F(VideoStreamEncoderTest,
7727 SwitchEncoderOnInitFailureWithoutEncoderSelector) {
7728 NiceMock<MockVideoEncoder> video_encoder;
7729 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7730 video_send_config_.encoder_settings.encoder_switch_request_callback =
7731 &switch_callback;
7732 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7733 &video_encoder, /*encoder_selector=*/nullptr);
7734 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7735
7736 // Reset encoder for new configuration to take effect.
7737 ConfigureEncoder(video_encoder_config_.Copy());
7738
7739 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7740 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7741 /*round_trip_time_ms=*/0,
7742 /*cwnd_reduce_ratio=*/0);
7743 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7744
7745 ON_CALL(video_encoder, InitEncode(_, _))
7746 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7747
7748 rtc::Event encode_attempted;
7749 EXPECT_CALL(switch_callback,
7750 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "VP8"),
7751 /*allow_default_fallback=*/true))
7752 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7753
7754 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
Markus Handell2cfc1af2022-08-19 08:16:48 +00007755 encode_attempted.Wait(TimeDelta::Seconds(3));
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007756
7757 AdvanceTime(TimeDelta::Zero());
7758
7759 video_stream_encoder_->Stop();
7760
7761 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007762 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007763 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7764 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007765 video_stream_encoder_.reset();
7766}
7767
Byoungchan Lee13fe3672022-04-06 10:44:42 +09007768TEST_F(VideoStreamEncoderTest, NullEncoderReturnSwitch) {
7769 // As a variant of EncoderSelectorBrokenEncoderSwitch, when a null
7770 // VideoEncoder is passed in encoder_factory, it checks whether
7771 // Codec Switch occurs without a crash.
7772 constexpr int kSufficientBitrateToNotDrop = 1000;
7773 constexpr int kDontCare = 100;
7774
7775 NiceMock<MockEncoderSelector> encoder_selector;
7776 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7777 video_send_config_.encoder_settings.encoder_switch_request_callback =
7778 &switch_callback;
7779 auto encoder_factory =
7780 std::make_unique<test::VideoEncoderNullableProxyFactory>(
7781 /*encoder=*/nullptr, &encoder_selector);
7782 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7783
7784 // Reset encoder for new configuration to take effect.
7785 ConfigureEncoder(video_encoder_config_.Copy());
7786 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7787 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7788 // not fail.
7789 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7790 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7791 /*stable_target_bitrate=*/
7792 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7793 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7794 /*fraction_lost=*/0,
7795 /*round_trip_time_ms=*/0,
7796 /*cwnd_reduce_ratio=*/0);
7797 ON_CALL(encoder_selector, OnEncoderBroken)
7798 .WillByDefault(Return(SdpVideoFormat("AV2")));
7799 rtc::Event encode_attempted;
7800 EXPECT_CALL(switch_callback,
7801 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7802 /*allow_default_fallback=*/_))
7803 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7804
7805 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
Markus Handell2cfc1af2022-08-19 08:16:48 +00007806 encode_attempted.Wait(TimeDelta::Seconds(3));
Byoungchan Lee13fe3672022-04-06 10:44:42 +09007807
7808 AdvanceTime(TimeDelta::Zero());
7809
7810 video_stream_encoder_->Stop();
7811
7812 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7813 // to it's factory, so in order for the encoder instance in the
7814 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7815 // reset the `video_stream_encoder_` here.
7816 video_stream_encoder_.reset();
7817}
7818
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007819TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007820 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007821 const int kFrameWidth = 320;
7822 const int kFrameHeight = 180;
7823
7824 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007825 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007826 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007827 /*target_bitrate=*/rate,
7828 /*stable_target_bitrate=*/rate,
7829 /*link_allocation=*/rate,
7830 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007831 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007832 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007833
7834 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007835 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007836 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7837 frame.set_rotation(kVideoRotation_270);
7838 video_source_.IncomingCapturedFrame(frame);
7839 WaitForEncodedFrame(timestamp_ms);
7840 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7841
7842 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007843 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007844 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007845 /*target_bitrate=*/new_stable_rate,
7846 /*stable_target_bitrate=*/new_stable_rate,
7847 /*link_allocation=*/rate,
7848 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007849 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007850 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007851 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7852 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7853 video_stream_encoder_->Stop();
7854}
7855
7856TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007857 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007858 const int kFrameWidth = 320;
7859 const int kFrameHeight = 180;
7860
7861 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007862 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007863 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007864 /*target_bitrate=*/rate,
7865 /*stable_target_bitrate=*/rate,
7866 /*link_allocation=*/rate,
7867 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007868 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007869 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007870
7871 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007872 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007873 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7874 frame.set_rotation(kVideoRotation_270);
7875 video_source_.IncomingCapturedFrame(frame);
7876 WaitForEncodedFrame(timestamp_ms);
7877 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7878
7879 // Set a higher target rate without changing the link_allocation. Should not
7880 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007881 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007882 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007883 /*target_bitrate=*/rate,
7884 /*stable_target_bitrate=*/new_stable_rate,
7885 /*link_allocation=*/rate,
7886 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007887 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007888 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007889 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7890 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7891 video_stream_encoder_->Stop();
7892}
7893
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007894TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01007895 test::ScopedKeyValueConfig field_trials(
7896 field_trials_,
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007897 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7898 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7899 const int kFramerateFps = 30;
7900 const int kWidth = 1920;
7901 const int kHeight = 1080;
7902 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7903 // Works on screenshare mode.
7904 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7905 // We rely on the automatic resolution adaptation, but we handle framerate
7906 // adaptation manually by mocking the stats proxy.
7907 video_source_.set_adaptation_enabled(true);
7908
7909 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007910 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007911 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007912 video_stream_encoder_->SetSource(&video_source_,
7913 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007914 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007915
7916 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7917 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7918
7919 // Pass enough frames with the full update to trigger animation detection.
7920 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007921 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007922 frame.set_ntp_time_ms(timestamp_ms);
7923 frame.set_timestamp_us(timestamp_ms * 1000);
7924 video_source_.IncomingCapturedFrame(frame);
7925 WaitForEncodedFrame(timestamp_ms);
7926 }
7927
7928 // Resolution should be limited.
7929 rtc::VideoSinkWants expected;
7930 expected.max_framerate_fps = kFramerateFps;
7931 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007932 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007933
7934 // Pass one frame with no known update.
7935 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007936 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007937 frame.set_ntp_time_ms(timestamp_ms);
7938 frame.set_timestamp_us(timestamp_ms * 1000);
7939 frame.clear_update_rect();
7940
7941 video_source_.IncomingCapturedFrame(frame);
7942 WaitForEncodedFrame(timestamp_ms);
7943
7944 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007945 EXPECT_THAT(video_source_.sink_wants(),
7946 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007947
7948 video_stream_encoder_->Stop();
7949}
7950
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007951TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7952 const int kWidth = 720; // 540p adapted down.
7953 const int kHeight = 405;
7954 const int kNumFrames = 3;
7955 // Works on screenshare mode.
7956 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7957 /*num_spatial_layers=*/2, /*screenshare=*/true);
7958
7959 video_source_.set_adaptation_enabled(true);
7960
7961 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007962 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007963
7964 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7965
7966 // Pass enough frames with the full update to trigger animation detection.
7967 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007968 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007969 frame.set_ntp_time_ms(timestamp_ms);
7970 frame.set_timestamp_us(timestamp_ms * 1000);
7971 video_source_.IncomingCapturedFrame(frame);
7972 WaitForEncodedFrame(timestamp_ms);
7973 }
7974
7975 video_stream_encoder_->Stop();
7976}
7977
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007978TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7979 const float downscale_factors[] = {4.0, 2.0, 1.0};
7980 const int number_layers =
7981 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7982 VideoEncoderConfig config;
Jonas Oreland80c87d72022-09-29 15:01:09 +02007983 webrtc::VideoEncoder::EncoderInfo encoder_info;
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007984 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7985 for (int i = 0; i < number_layers; ++i) {
7986 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7987 config.simulcast_layers[i].active = true;
7988 }
7989 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007990 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007991 "VP8", /*max qp*/ 56, /*screencast*/ false,
Jonas Oreland80c87d72022-09-29 15:01:09 +02007992 /*screenshare enabled*/ false, encoder_info);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007993 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007994 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7995 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007996
7997 // First initialization.
7998 // Encoder should be initialized. Next frame should be key frame.
7999 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8000 sink_.SetNumExpectedLayers(number_layers);
8001 int64_t timestamp_ms = kFrameIntervalMs;
8002 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8003 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008004 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008005 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8006 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
8007 VideoFrameType::kVideoFrameKey,
8008 VideoFrameType::kVideoFrameKey}));
8009
8010 // Disable top layer.
8011 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
8012 config.simulcast_layers[number_layers - 1].active = false;
8013 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8014 sink_.SetNumExpectedLayers(number_layers - 1);
8015 timestamp_ms += kFrameIntervalMs;
8016 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8017 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008018 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008019 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8020 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
8021 VideoFrameType::kVideoFrameDelta,
8022 VideoFrameType::kVideoFrameDelta}));
8023
8024 // Re-enable top layer.
8025 // Encoder should be re-initialized. Next frame should be key frame.
8026 config.simulcast_layers[number_layers - 1].active = true;
8027 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8028 sink_.SetNumExpectedLayers(number_layers);
8029 timestamp_ms += kFrameIntervalMs;
8030 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8031 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008032 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008033 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8034 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
8035 VideoFrameType::kVideoFrameKey,
8036 VideoFrameType::kVideoFrameKey}));
8037
8038 // Top layer max rate change.
8039 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
8040 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
8041 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8042 sink_.SetNumExpectedLayers(number_layers);
8043 timestamp_ms += kFrameIntervalMs;
8044 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8045 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008046 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008047 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8048 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
8049 VideoFrameType::kVideoFrameDelta,
8050 VideoFrameType::kVideoFrameDelta}));
8051
8052 // Top layer resolution change.
8053 // Encoder should be re-initialized. Next frame should be key frame.
8054 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
8055 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8056 sink_.SetNumExpectedLayers(number_layers);
8057 timestamp_ms += kFrameIntervalMs;
8058 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8059 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008060 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008061 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8062 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
8063 VideoFrameType::kVideoFrameKey,
8064 VideoFrameType::kVideoFrameKey}));
8065 video_stream_encoder_->Stop();
8066}
8067
Henrik Boström1124ed12021-02-25 10:30:39 +01008068TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
8069 const int kFrameWidth = 1280;
8070 const int kFrameHeight = 720;
8071
8072 SetUp();
8073 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008074 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01008075
8076 // Capturing a frame should reconfigure the encoder and expose the encoder
8077 // resolution, which is the same as the input frame.
8078 int64_t timestamp_ms = kFrameIntervalMs;
8079 video_source_.IncomingCapturedFrame(
8080 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8081 WaitForEncodedFrame(timestamp_ms);
8082 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8083 EXPECT_THAT(video_source_.sink_wants().resolutions,
8084 ::testing::ElementsAreArray(
8085 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
8086
8087 video_stream_encoder_->Stop();
8088}
8089
8090TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
8091 // Pick downscale factors such that we never encode at full resolution - this
8092 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 16:22:11 +02008093 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 10:30:39 +01008094 // encoder should not ask for the frame resolution. This allows video frames
8095 // to have the appearence of one resolution but optimize its internal buffers
8096 // for what is actually encoded.
8097 const size_t kNumSimulcastLayers = 3u;
8098 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
8099 const int kFrameWidth = 1280;
8100 const int kFrameHeight = 720;
8101 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8102 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8103 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8104 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8105 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8106 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8107
8108 VideoEncoderConfig config;
Jonas Oreland80c87d72022-09-29 15:01:09 +02008109 webrtc::VideoEncoder::EncoderInfo encoder_info;
Henrik Boström1124ed12021-02-25 10:30:39 +01008110 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
8111 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
8112 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
8113 config.simulcast_layers[i].active = true;
8114 }
8115 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02008116 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01008117 "VP8", /*max qp*/ 56, /*screencast*/ false,
Jonas Oreland80c87d72022-09-29 15:01:09 +02008118 /*screenshare enabled*/ false, encoder_info);
Henrik Boström1124ed12021-02-25 10:30:39 +01008119 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008120 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8121 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01008122
8123 // Capture a frame with all layers active.
8124 int64_t timestamp_ms = kFrameIntervalMs;
8125 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
8126 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8127 video_source_.IncomingCapturedFrame(
8128 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8129 WaitForEncodedFrame(timestamp_ms);
8130 // Expect encoded resolutions to match the expected simulcast layers.
8131 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8132 EXPECT_THAT(
8133 video_source_.sink_wants().resolutions,
8134 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
8135
8136 // Capture a frame with one of the layers inactive.
8137 timestamp_ms += kFrameIntervalMs;
8138 config.simulcast_layers[2].active = false;
8139 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
8140 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8141 video_source_.IncomingCapturedFrame(
8142 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8143 WaitForEncodedFrame(timestamp_ms);
8144
8145 // Expect encoded resolutions to match the expected simulcast layers.
8146 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8147 EXPECT_THAT(video_source_.sink_wants().resolutions,
8148 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
8149
8150 // Capture a frame with all but one layer turned off.
8151 timestamp_ms += kFrameIntervalMs;
8152 config.simulcast_layers[1].active = false;
8153 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
8154 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8155 video_source_.IncomingCapturedFrame(
8156 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8157 WaitForEncodedFrame(timestamp_ms);
8158
8159 // Expect encoded resolutions to match the expected simulcast layers.
8160 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8161 EXPECT_THAT(video_source_.sink_wants().resolutions,
8162 ::testing::ElementsAreArray({kLayer0Size}));
8163
8164 video_stream_encoder_->Stop();
8165}
8166
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008167TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008168 ResetEncoder("VP8", 1, 1, 1, false);
8169
Niels Möller8b692902021-06-14 12:04:57 +02008170 // Force encoder reconfig.
8171 video_source_.IncomingCapturedFrame(
8172 CreateFrame(1, codec_width_, codec_height_));
8173 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8174
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008175 // Set QP on encoded frame and pass the frame to encode complete callback.
8176 // Since QP is present QP parsing won't be triggered and the original value
8177 // should be kept.
8178 EncodedImage encoded_image;
8179 encoded_image.qp_ = 123;
8180 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8181 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8182 CodecSpecificInfo codec_info;
8183 codec_info.codecType = kVideoCodecVP8;
8184 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
Markus Handell2cfc1af2022-08-19 08:16:48 +00008185 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeout));
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008186 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
8187 video_stream_encoder_->Stop();
8188}
8189
8190TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008191 ResetEncoder("VP8", 1, 1, 1, false);
8192
Niels Möller8b692902021-06-14 12:04:57 +02008193 // Force encoder reconfig.
8194 video_source_.IncomingCapturedFrame(
8195 CreateFrame(1, codec_width_, codec_height_));
8196 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8197
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008198 // Pass an encoded frame without QP to encode complete callback. QP should be
8199 // parsed and set.
8200 EncodedImage encoded_image;
8201 encoded_image.qp_ = -1;
8202 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8203 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8204 CodecSpecificInfo codec_info;
8205 codec_info.codecType = kVideoCodecVP8;
8206 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
Markus Handell2cfc1af2022-08-19 08:16:48 +00008207 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeout));
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008208 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
8209 video_stream_encoder_->Stop();
8210}
8211
8212TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01008213 webrtc::test::ScopedKeyValueConfig field_trials(
8214 field_trials_, "WebRTC-QpParsingKillSwitch/Enabled/");
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008215
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008216 ResetEncoder("VP8", 1, 1, 1, false);
8217
Niels Möller8b692902021-06-14 12:04:57 +02008218 // Force encoder reconfig.
8219 video_source_.IncomingCapturedFrame(
8220 CreateFrame(1, codec_width_, codec_height_));
8221 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8222
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008223 EncodedImage encoded_image;
8224 encoded_image.qp_ = -1;
8225 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8226 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8227 CodecSpecificInfo codec_info;
8228 codec_info.codecType = kVideoCodecVP8;
8229 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
Markus Handell2cfc1af2022-08-19 08:16:48 +00008230 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeout));
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008231 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
8232 video_stream_encoder_->Stop();
8233}
8234
Sergey Silkind19e3b92021-03-16 10:05:30 +00008235TEST_F(VideoStreamEncoderTest,
8236 QualityScalingNotAllowed_QualityScalingDisabled) {
8237 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8238
8239 // Disable scaling settings in encoder info.
8240 fake_encoder_.SetQualityScaling(false);
8241 // Disable quality scaling in encoder config.
8242 video_encoder_config.is_quality_scaling_allowed = false;
8243 ConfigureEncoder(std::move(video_encoder_config));
8244
8245 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008246 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008247
8248 test::FrameForwarder source;
8249 video_stream_encoder_->SetSource(
8250 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8251 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8252 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8253
8254 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8255 WaitForEncodedFrame(1);
8256 video_stream_encoder_->TriggerQualityLow();
8257 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8258
8259 video_stream_encoder_->Stop();
8260}
8261
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008262TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
8263 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8264
8265 // Disable scaling settings in encoder info.
8266 fake_encoder_.SetQualityScaling(false);
8267 // Set QP trusted in encoder info.
8268 fake_encoder_.SetIsQpTrusted(true);
8269 // Enable quality scaling in encoder config.
8270 video_encoder_config.is_quality_scaling_allowed = false;
8271 ConfigureEncoder(std::move(video_encoder_config));
8272
8273 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008274 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008275
8276 test::FrameForwarder source;
8277 video_stream_encoder_->SetSource(
8278 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8279 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8280 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8281
8282 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8283 WaitForEncodedFrame(1);
8284 video_stream_encoder_->TriggerQualityLow();
8285 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8286
8287 video_stream_encoder_->Stop();
8288}
8289
Shuhai Pengf2707702021-09-29 17:19:44 +08008290TEST_F(VideoStreamEncoderTest,
8291 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8292 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8293
8294 // Disable scaling settings in encoder info.
8295 fake_encoder_.SetQualityScaling(false);
8296 // Set QP trusted in encoder info.
8297 fake_encoder_.SetIsQpTrusted(true);
8298 // Enable quality scaling in encoder config.
8299 video_encoder_config.is_quality_scaling_allowed = false;
8300 ConfigureEncoder(std::move(video_encoder_config));
8301
8302 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008303 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008304
8305 test::FrameForwarder source;
8306 video_stream_encoder_->SetSource(
8307 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8308 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8309 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8310
8311 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8312 WaitForEncodedFrame(1);
8313 video_stream_encoder_->TriggerQualityLow();
8314 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8315
8316 video_stream_encoder_->Stop();
8317}
8318
8319TEST_F(VideoStreamEncoderTest,
8320 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8321 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8322
8323 // Disable scaling settings in encoder info.
8324 fake_encoder_.SetQualityScaling(false);
8325 // Set QP trusted in encoder info.
8326 fake_encoder_.SetIsQpTrusted(false);
8327 // Enable quality scaling in encoder config.
8328 video_encoder_config.is_quality_scaling_allowed = false;
8329 ConfigureEncoder(std::move(video_encoder_config));
8330
8331 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008332 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008333
8334 test::FrameForwarder source;
8335 video_stream_encoder_->SetSource(
8336 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8337 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8338 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8339
8340 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8341 WaitForEncodedFrame(1);
8342 video_stream_encoder_->TriggerQualityLow();
8343 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8344
8345 video_stream_encoder_->Stop();
8346}
8347
8348TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8349 // Set QP trusted in encoder info.
8350 fake_encoder_.SetIsQpTrusted(false);
8351
8352 const int MinEncBitrateKbps = 30;
8353 const int MaxEncBitrateKbps = 100;
8354 const int MinStartBitrateKbp = 50;
8355 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8356 /*frame_size_pixels=*/codec_width_ * codec_height_,
8357 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8358 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8359 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8360
8361 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008362 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008363
8364 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8365
8366 VideoEncoderConfig video_encoder_config;
8367 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8368 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8369 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8370 MinEncBitrateKbps * 1000;
8371 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8372 kMaxPayloadLength);
8373
8374 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8375 WaitForEncodedFrame(1);
8376 EXPECT_EQ(
8377 MaxEncBitrateKbps,
8378 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8379 EXPECT_EQ(
8380 MinEncBitrateKbps,
8381 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8382
8383 video_stream_encoder_->Stop();
8384}
8385
8386TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8387 // Set QP trusted in encoder info.
8388 fake_encoder_.SetIsQpTrusted(false);
8389
8390 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8391 EncoderInfoSettings::
8392 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8393 codec_width_ * codec_height_,
8394 EncoderInfoSettings::
8395 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8396 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8397
8398 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8399 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8400 const int TargetEncBitrate = MaxEncBitrate;
8401 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8402 DataRate::BitsPerSec(TargetEncBitrate),
8403 DataRate::BitsPerSec(TargetEncBitrate),
8404 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8405
8406 VideoEncoderConfig video_encoder_config;
8407 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8408 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8409 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8410 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8411 kMaxPayloadLength);
8412
8413 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8414 WaitForEncodedFrame(1);
8415 EXPECT_EQ(
8416 MaxEncBitrate / 1000,
8417 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8418 EXPECT_EQ(
8419 MinEncBitrate / 1000,
8420 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8421
8422 video_stream_encoder_->Stop();
8423}
8424
Erik Språnge4589cb2022-04-06 16:44:30 +02008425TEST_F(VideoStreamEncoderTest, NormalComplexityWithMoreThanTwoCores) {
8426 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8427 /*num_spatial_layers=*/1,
8428 /*screenshare=*/false, /*allocation_callback_type=*/
8429 VideoStreamEncoder::BitrateAllocationCallbackType::
8430 kVideoBitrateAllocationWhenScreenSharing,
8431 /*num_cores=*/3);
8432
8433 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8434 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8435 video_source_.IncomingCapturedFrame(
8436 CreateFrame(1, /*width=*/320, /*height=*/180));
8437 WaitForEncodedFrame(1);
8438 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8439 VideoCodecComplexity::kComplexityNormal);
8440 video_stream_encoder_->Stop();
8441}
8442
8443TEST_F(VideoStreamEncoderTest,
8444 NormalComplexityWhenLowTierOptimizationsAreDisabled) {
8445 webrtc::test::ScopedKeyValueConfig field_trials(
8446 field_trials_, "WebRTC-VP9-LowTierOptimizations/Disabled/");
8447
8448 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8449 /*num_spatial_layers=*/1,
8450 /*screenshare=*/false, /*allocation_callback_type=*/
8451 VideoStreamEncoder::BitrateAllocationCallbackType::
8452 kVideoBitrateAllocationWhenScreenSharing,
8453 /*num_cores=*/2);
8454
8455 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8456 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8457 video_source_.IncomingCapturedFrame(
8458 CreateFrame(1, /*width=*/320, /*height=*/180));
8459 WaitForEncodedFrame(1);
8460 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8461 VideoCodecComplexity::kComplexityNormal);
8462 video_stream_encoder_->Stop();
8463}
8464
8465TEST_F(VideoStreamEncoderTest, LowComplexityWithTwoCores) {
8466 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8467 /*num_spatial_layers=*/1,
8468 /*screenshare=*/false, /*allocation_callback_type=*/
8469 VideoStreamEncoder::BitrateAllocationCallbackType::
8470 kVideoBitrateAllocationWhenScreenSharing,
8471 /*num_cores=*/2);
8472
8473 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8474 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8475 video_source_.IncomingCapturedFrame(
8476 CreateFrame(1, /*width=*/320, /*height=*/180));
8477 WaitForEncodedFrame(1);
8478 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8479 VideoCodecComplexity::kComplexityLow);
8480 video_stream_encoder_->Stop();
8481}
8482
Sergey Silkind19e3b92021-03-16 10:05:30 +00008483#if !defined(WEBRTC_IOS)
8484// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8485// disabled by default on iOS.
8486TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8487 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8488
8489 // Disable scaling settings in encoder info.
8490 fake_encoder_.SetQualityScaling(false);
8491 // Enable quality scaling in encoder config.
8492 video_encoder_config.is_quality_scaling_allowed = true;
8493 ConfigureEncoder(std::move(video_encoder_config));
8494
8495 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008496 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008497
8498 test::FrameForwarder source;
8499 video_stream_encoder_->SetSource(
8500 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8501 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8502 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8503
8504 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8505 WaitForEncodedFrame(1);
8506 video_stream_encoder_->TriggerQualityLow();
8507 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8508
8509 video_stream_encoder_->Stop();
8510}
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008511
8512TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8513 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8514
8515 // Disable scaling settings in encoder info.
8516 fake_encoder_.SetQualityScaling(false);
8517 // Set QP trusted in encoder info.
8518 fake_encoder_.SetIsQpTrusted(true);
8519 // Enable quality scaling in encoder config.
8520 video_encoder_config.is_quality_scaling_allowed = true;
8521 ConfigureEncoder(std::move(video_encoder_config));
8522
8523 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008524 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008525
8526 test::FrameForwarder source;
8527 video_stream_encoder_->SetSource(
8528 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8529 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8530 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8531
8532 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8533 WaitForEncodedFrame(1);
8534 video_stream_encoder_->TriggerQualityLow();
8535 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8536
8537 video_stream_encoder_->Stop();
8538}
Shuhai Pengf2707702021-09-29 17:19:44 +08008539
8540TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8541 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8542
8543 // Disable scaling settings in encoder info.
8544 fake_encoder_.SetQualityScaling(false);
8545 // Set QP not trusted in encoder info.
8546 fake_encoder_.SetIsQpTrusted(false);
8547 // Enable quality scaling in encoder config.
8548 video_encoder_config.is_quality_scaling_allowed = true;
8549 ConfigureEncoder(std::move(video_encoder_config));
8550
8551 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008552 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008553
8554 test::FrameForwarder source;
8555 video_stream_encoder_->SetSource(
8556 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8557 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8558 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8559
8560 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8561 WaitForEncodedFrame(1);
8562 video_stream_encoder_->TriggerQualityLow();
8563 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8564 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8565 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8566
8567 video_stream_encoder_->Stop();
8568}
8569
8570TEST_F(VideoStreamEncoderTest,
8571 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8572 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8573
8574 // Disable scaling settings in encoder info.
8575 fake_encoder_.SetQualityScaling(false);
8576 // Set QP trusted in encoder info.
8577 fake_encoder_.SetIsQpTrusted(true);
8578 // Enable quality scaling in encoder config.
8579 video_encoder_config.is_quality_scaling_allowed = true;
8580 ConfigureEncoder(std::move(video_encoder_config));
8581
8582 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008583 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008584
8585 test::FrameForwarder source;
8586 video_stream_encoder_->SetSource(
8587 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8588 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8589 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8590
8591 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8592 WaitForEncodedFrame(1);
8593 video_stream_encoder_->TriggerQualityLow();
8594 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8595 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8596
8597 video_stream_encoder_->Stop();
8598}
8599
8600TEST_F(VideoStreamEncoderTest,
8601 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8602 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8603
8604 // Disable scaling settings in encoder info.
8605 fake_encoder_.SetQualityScaling(false);
8606 // Set QP trusted in encoder info.
8607 fake_encoder_.SetIsQpTrusted(false);
8608 // Enable quality scaling in encoder config.
8609 video_encoder_config.is_quality_scaling_allowed = true;
8610 ConfigureEncoder(std::move(video_encoder_config));
8611
8612 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008613 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008614
8615 test::FrameForwarder source;
8616 video_stream_encoder_->SetSource(
8617 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8618 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8619 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8620
8621 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8622 WaitForEncodedFrame(1);
8623 video_stream_encoder_->TriggerQualityLow();
8624 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8625
8626 video_stream_encoder_->Stop();
8627}
8628
Erik Språng5e13d052022-08-02 11:42:49 +02008629TEST_F(VideoStreamEncoderTest,
8630 RequestsRefreshFrameAfterEarlyDroppedNativeFrame) {
8631 // Send a native frame before encoder rates have been set. The encoder is
8632 // seen as paused at this time.
8633 rtc::Event frame_destroyed_event;
8634 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
8635 /*ntp_time_ms=*/1, &frame_destroyed_event, codec_width_, codec_height_));
8636
8637 // Frame should be dropped and destroyed.
8638 ExpectDroppedFrame();
Markus Handell2cfc1af2022-08-19 08:16:48 +00008639 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
Erik Språng5e13d052022-08-02 11:42:49 +02008640 EXPECT_EQ(video_source_.refresh_frames_requested_, 0);
8641
8642 // Set bitrates, unpausing the encoder and triggering a request for a refresh
8643 // frame.
8644 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8645 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8646 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8647 EXPECT_EQ(video_source_.refresh_frames_requested_, 1);
8648
8649 video_stream_encoder_->Stop();
8650}
8651
Erik Språnge4589cb2022-04-06 16:44:30 +02008652#endif // !defined(WEBRTC_IOS)
Sergey Silkind19e3b92021-03-16 10:05:30 +00008653
Henrik Boström56db9ff2021-03-24 09:06:45 +01008654// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8655class VideoStreamEncoderWithRealEncoderTest
8656 : public VideoStreamEncoderTest,
8657 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8658 public:
8659 VideoStreamEncoderWithRealEncoderTest()
8660 : VideoStreamEncoderTest(),
8661 codec_type_(std::get<0>(GetParam())),
8662 allow_i420_conversion_(std::get<1>(GetParam())) {}
8663
8664 void SetUp() override {
8665 VideoStreamEncoderTest::SetUp();
8666 std::unique_ptr<VideoEncoder> encoder;
8667 switch (codec_type_) {
8668 case kVideoCodecVP8:
8669 encoder = VP8Encoder::Create();
8670 break;
8671 case kVideoCodecVP9:
8672 encoder = VP9Encoder::Create();
8673 break;
8674 case kVideoCodecAV1:
philipel09a28482022-05-25 09:47:06 +02008675 encoder = CreateLibaomAv1Encoder();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008676 break;
8677 case kVideoCodecH264:
8678 encoder =
8679 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8680 break;
8681 case kVideoCodecMultiplex:
8682 mock_encoder_factory_for_multiplex_ =
8683 std::make_unique<MockVideoEncoderFactory>();
8684 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8685 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8686 .WillRepeatedly([] { return VP8Encoder::Create(); });
8687 encoder = std::make_unique<MultiplexEncoderAdapter>(
8688 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8689 false);
8690 break;
8691 default:
Artem Titovd3251962021-11-15 16:57:07 +01008692 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008693 }
8694 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8695 }
8696
8697 void TearDown() override {
8698 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 16:22:11 +02008699 // Ensure `video_stream_encoder_` is destroyed before
8700 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 09:06:45 +01008701 video_stream_encoder_.reset();
8702 VideoStreamEncoderTest::TearDown();
8703 }
8704
8705 protected:
8706 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8707 std::unique_ptr<VideoEncoder> encoder) {
8708 // Configure VSE to use the encoder.
8709 encoder_ = std::move(encoder);
8710 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8711 encoder_.get(), &encoder_selector_);
8712 video_send_config_.encoder_settings.encoder_factory =
8713 encoder_proxy_factory_.get();
8714 VideoEncoderConfig video_encoder_config;
8715 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8716 video_encoder_config_ = video_encoder_config.Copy();
8717 ConfigureEncoder(video_encoder_config_.Copy());
8718
8719 // Set bitrate to ensure frame is not dropped.
8720 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008721 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008722 }
8723
8724 const VideoCodecType codec_type_;
8725 const bool allow_i420_conversion_;
8726 NiceMock<MockEncoderSelector> encoder_selector_;
8727 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8728 std::unique_ptr<VideoEncoder> encoder_;
8729 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8730};
8731
8732TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8733 auto native_i420_frame = test::CreateMappableNativeFrame(
8734 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8735 video_source_.IncomingCapturedFrame(native_i420_frame);
8736 WaitForEncodedFrame(codec_width_, codec_height_);
8737
8738 auto mappable_native_buffer =
8739 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8740 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8741 mappable_native_buffer->GetMappedFramedBuffers();
8742 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8743 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8744 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8745 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8746}
8747
8748TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8749 auto native_nv12_frame = test::CreateMappableNativeFrame(
8750 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8751 video_source_.IncomingCapturedFrame(native_nv12_frame);
8752 WaitForEncodedFrame(codec_width_, codec_height_);
8753
8754 auto mappable_native_buffer =
8755 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8756 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8757 mappable_native_buffer->GetMappedFramedBuffers();
8758 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8759 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8760 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8761 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8762
8763 if (!allow_i420_conversion_) {
8764 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8765 }
8766}
8767
Erik Språng7444b192021-06-02 14:02:13 +02008768TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8769 if (codec_type_ == kVideoCodecMultiplex) {
8770 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8771 return;
8772 }
8773
8774 const size_t kNumSpatialLayers = 3u;
8775 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8776 const int kFrameWidth = 1280;
8777 const int kFrameHeight = 720;
8778 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8779 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8780 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8781 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8782 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8783 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8784
8785 VideoEncoderConfig config;
Jonas Oreland80c87d72022-09-29 15:01:09 +02008786 webrtc::VideoEncoder::EncoderInfo encoder_info;
Erik Språng7444b192021-06-02 14:02:13 +02008787 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8788 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008789 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008790 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8791 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8792 vp9_settings.numberOfTemporalLayers = 3;
8793 vp9_settings.automaticResizeOn = false;
8794 config.encoder_specific_settings =
8795 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8796 vp9_settings);
8797 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8798 /*fps=*/30.0,
8799 /*first_active_layer=*/0,
8800 /*num_spatial_layers=*/3,
8801 /*num_temporal_layers=*/3,
8802 /*is_screenshare=*/false);
8803 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8804 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008805 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008806 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8807 /*fps=*/30.0,
8808 /*first_active_layer=*/0,
8809 /*num_spatial_layers=*/3,
8810 /*num_temporal_layers=*/3,
8811 /*is_screenshare=*/false);
Niels Möller79d566b2022-04-29 11:03:13 +02008812 config.simulcast_layers[0].scalability_mode = ScalabilityMode::kL3T3_KEY;
Erik Språng7444b192021-06-02 14:02:13 +02008813 } else {
8814 // Simulcast for VP8/H264.
8815 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8816 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8817 config.simulcast_layers[i].scale_resolution_down_by =
8818 kDownscaleFactors[i];
8819 config.simulcast_layers[i].active = true;
8820 }
8821 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8822 // Turn off frame dropping to prevent flakiness.
Niels Möller807328f2022-05-12 16:16:39 +02008823 config.frame_drop_enabled = false;
Erik Språng7444b192021-06-02 14:02:13 +02008824 }
8825 }
8826
8827 auto set_layer_active = [&](int layer_idx, bool active) {
8828 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8829 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8830 config.spatial_layers[layer_idx].active = active;
8831 } else {
8832 config.simulcast_layers[layer_idx].active = active;
8833 }
8834 };
8835
8836 config.video_stream_factory =
8837 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8838 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8839 /*screencast*/ false,
Jonas Oreland80c87d72022-09-29 15:01:09 +02008840 /*screenshare enabled*/ false, encoder_info);
Erik Språng7444b192021-06-02 14:02:13 +02008841 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008842 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8843 0, 0, 0);
Erik Språng7444b192021-06-02 14:02:13 +02008844
8845 // Capture a frame with all layers active.
8846 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8847 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8848 int64_t timestamp_ms = kFrameIntervalMs;
8849 video_source_.IncomingCapturedFrame(
8850 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8851
8852 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8853 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8854
8855 // Capture a frame with one of the layers inactive.
8856 set_layer_active(2, false);
8857 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8858 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8859 timestamp_ms += kFrameIntervalMs;
8860 video_source_.IncomingCapturedFrame(
8861 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8862 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8863
8864 // New target bitrates signaled based on lower resolution.
8865 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8866 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8867 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8868 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8869
8870 // Re-enable the top layer.
8871 set_layer_active(2, true);
8872 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8873 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8874 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8875
8876 // Bitrate target adjusted back up to enable HD layer...
8877 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8878 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8879 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8880 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8881
8882 // ...then add a new frame.
8883 timestamp_ms += kFrameIntervalMs;
8884 video_source_.IncomingCapturedFrame(
8885 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8886 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8887 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8888
8889 video_stream_encoder_->Stop();
8890}
8891
Henrik Boström56db9ff2021-03-24 09:06:45 +01008892std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8893 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8894 VideoCodecType codec_type = std::get<0>(info.param);
8895 bool allow_i420_conversion = std::get<1>(info.param);
8896 std::string str;
8897 switch (codec_type) {
8898 case kVideoCodecGeneric:
8899 str = "Generic";
8900 break;
8901 case kVideoCodecVP8:
8902 str = "VP8";
8903 break;
8904 case kVideoCodecVP9:
8905 str = "VP9";
8906 break;
8907 case kVideoCodecAV1:
8908 str = "AV1";
8909 break;
8910 case kVideoCodecH264:
8911 str = "H264";
8912 break;
8913 case kVideoCodecMultiplex:
8914 str = "Multiplex";
8915 break;
8916 default:
Artem Titovd3251962021-11-15 16:57:07 +01008917 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008918 }
8919 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8920 return str;
8921}
8922
8923constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8924 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8925constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8926 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8927constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
Ilya Nikolaevskiy1bcdafc2022-03-09 16:01:07 +01008928 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/false);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008929constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8930 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8931#if defined(WEBRTC_USE_H264)
8932constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8933 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8934
8935// The windows compiler does not tolerate #if statements inside the
8936// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8937// and without H264).
8938INSTANTIATE_TEST_SUITE_P(
8939 All,
8940 VideoStreamEncoderWithRealEncoderTest,
8941 ::testing::Values(kVP8DisallowConversion,
8942 kVP9DisallowConversion,
8943 kAV1AllowConversion,
8944 kMultiplexDisallowConversion,
8945 kH264AllowConversion),
8946 TestParametersVideoCodecAndAllowI420ConversionToString);
8947#else
8948INSTANTIATE_TEST_SUITE_P(
8949 All,
8950 VideoStreamEncoderWithRealEncoderTest,
8951 ::testing::Values(kVP8DisallowConversion,
8952 kVP9DisallowConversion,
8953 kAV1AllowConversion,
8954 kMultiplexDisallowConversion),
8955 TestParametersVideoCodecAndAllowI420ConversionToString);
8956#endif
8957
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008958class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8959 protected:
8960 void RunTest(const std::vector<VideoStream>& configs,
8961 const int expected_num_init_encode) {
8962 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008963 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008964 InsertFrameAndWaitForEncoded();
8965 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8966 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008967 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8968 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008969
8970 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8971 ConfigureEncoder(configs[1]);
8972 InsertFrameAndWaitForEncoded();
8973 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8974 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008975 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008976 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 10:07:11 +02008977 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008978
8979 video_stream_encoder_->Stop();
8980 }
8981
8982 void ConfigureEncoder(const VideoStream& stream) {
8983 VideoEncoderConfig config;
Jonas Oreland80c87d72022-09-29 15:01:09 +02008984 webrtc::VideoEncoder::EncoderInfo encoder_info;
8985
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008986 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8987 config.max_bitrate_bps = stream.max_bitrate_bps;
8988 config.simulcast_layers[0] = stream;
8989 config.video_stream_factory =
8990 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8991 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
Jonas Oreland80c87d72022-09-29 15:01:09 +02008992 /*conference_mode=*/false, encoder_info);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008993 video_stream_encoder_->ConfigureEncoder(std::move(config),
8994 kMaxPayloadLength);
8995 }
8996
8997 void OnBitrateUpdated(DataRate bitrate) {
8998 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8999 bitrate, bitrate, bitrate, 0, 0, 0);
9000 }
9001
9002 void InsertFrameAndWaitForEncoded() {
9003 timestamp_ms_ += kFrameIntervalMs;
9004 video_source_.IncomingCapturedFrame(
9005 CreateFrame(timestamp_ms_, kWidth, kHeight));
9006 sink_.WaitForEncodedFrame(timestamp_ms_);
9007 }
9008
9009 void ExpectEqual(const VideoCodec& actual,
9010 const VideoStream& expected) const {
9011 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
9012 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
9013 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
9014 static_cast<unsigned int>(expected.min_bitrate_bps));
9015 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
9016 static_cast<unsigned int>(expected.max_bitrate_bps));
9017 EXPECT_EQ(actual.simulcastStream[0].width,
9018 kWidth / expected.scale_resolution_down_by);
9019 EXPECT_EQ(actual.simulcastStream[0].height,
9020 kHeight / expected.scale_resolution_down_by);
9021 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
9022 expected.num_temporal_layers);
Niels Möller79d566b2022-04-29 11:03:13 +02009023 EXPECT_EQ(actual.GetScalabilityMode(), expected.scalability_mode);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02009024 }
9025
9026 VideoStream DefaultConfig() const {
9027 VideoStream stream;
9028 stream.max_framerate = 25;
9029 stream.min_bitrate_bps = 35000;
9030 stream.max_bitrate_bps = 900000;
9031 stream.scale_resolution_down_by = 1.0;
9032 stream.num_temporal_layers = 1;
9033 stream.bitrate_priority = 1.0;
Niels Möller79d566b2022-04-29 11:03:13 +02009034 stream.scalability_mode = absl::nullopt;
Åsa Persson4d4f62f2021-09-12 16:14:48 +02009035 return stream;
9036 }
9037
9038 const int kWidth = 640;
9039 const int kHeight = 360;
9040 int64_t timestamp_ms_ = 0;
9041};
9042
9043TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
9044 VideoStream config1 = DefaultConfig();
9045 VideoStream config2 = config1;
9046 config2.max_framerate++;
9047
9048 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9049}
9050
9051TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
9052 VideoStream config1 = DefaultConfig();
9053 VideoStream config2 = config1;
9054 config2.min_bitrate_bps += 10000;
9055
9056 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9057}
9058
9059TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
9060 VideoStream config1 = DefaultConfig();
9061 VideoStream config2 = config1;
9062 config2.max_bitrate_bps += 100000;
9063
9064 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9065}
9066
9067TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
9068 VideoStream config1 = DefaultConfig();
9069 VideoStream config2 = config1;
9070 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
9071
9072 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9073}
9074
9075TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
9076 VideoStream config1 = DefaultConfig();
9077 VideoStream config2 = config1;
9078 config2.scale_resolution_down_by *= 2;
9079
9080 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9081}
9082
9083TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
9084 VideoStream config1 = DefaultConfig();
9085 VideoStream config2 = config1;
9086 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
9087
9088 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9089}
9090
9091TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
9092 VideoStream config1 = DefaultConfig();
9093 VideoStream config2 = config1;
Niels Möller05954892022-05-13 13:34:37 +02009094 config2.scalability_mode = ScalabilityMode::kL2T1;
9095
9096 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9097}
9098
9099TEST_F(ReconfigureEncoderTest,
9100 UpdatesNumTemporalLayersFromScalabilityModeChanges) {
9101 VideoStream config1 = DefaultConfig();
9102 VideoStream config2 = config1;
Niels Möller79d566b2022-04-29 11:03:13 +02009103 config2.scalability_mode = ScalabilityMode::kL1T2;
Niels Möller05954892022-05-13 13:34:37 +02009104 config2.num_temporal_layers = 2;
Åsa Persson4d4f62f2021-09-12 16:14:48 +02009105
9106 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9107}
9108
Tommi62b01db2022-01-25 23:41:22 +01009109// Simple test that just creates and then immediately destroys an encoder.
9110// The purpose of the test is to make sure that nothing bad happens if the
9111// initialization step on the encoder queue, doesn't run.
9112TEST(VideoStreamEncoderSimpleTest, CreateDestroy) {
9113 class SuperLazyTaskQueue : public webrtc::TaskQueueBase {
9114 public:
9115 SuperLazyTaskQueue() = default;
9116 ~SuperLazyTaskQueue() override = default;
9117
9118 private:
9119 void Delete() override { delete this; }
Danil Chapovalov95eeaa72022-07-06 10:14:29 +02009120 void PostTask(absl::AnyInvocable<void() &&> task) override {
Tommi62b01db2022-01-25 23:41:22 +01009121 // meh.
9122 }
Danil Chapovalov95eeaa72022-07-06 10:14:29 +02009123 void PostDelayedTask(absl::AnyInvocable<void() &&> task,
9124 TimeDelta delay) override {
Tommi62b01db2022-01-25 23:41:22 +01009125 ASSERT_TRUE(false);
9126 }
Danil Chapovalov95eeaa72022-07-06 10:14:29 +02009127 void PostDelayedHighPrecisionTask(absl::AnyInvocable<void() &&> task,
9128 TimeDelta delay) override {
9129 ADD_FAILURE();
9130 }
Tommi62b01db2022-01-25 23:41:22 +01009131 };
9132
9133 // Lots of boiler plate.
Jonas Oreland8ca06132022-03-14 12:52:48 +01009134 test::ScopedKeyValueConfig field_trials;
Philipp Hanckea204ad22022-07-08 18:43:25 +02009135 GlobalSimulatedTimeController time_controller(Timestamp::Zero());
Tommi62b01db2022-01-25 23:41:22 +01009136 auto stats_proxy = std::make_unique<MockableSendStatisticsProxy>(
9137 time_controller.GetClock(), VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 12:52:48 +01009138 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo, field_trials);
Tommi62b01db2022-01-25 23:41:22 +01009139 SimpleVideoStreamEncoderFactory::MockFakeEncoder mock_fake_encoder(
9140 time_controller.GetClock());
9141 test::VideoEncoderProxyFactory encoder_factory(&mock_fake_encoder);
9142 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory =
9143 CreateBuiltinVideoBitrateAllocatorFactory();
9144 VideoStreamEncoderSettings encoder_settings{
9145 VideoEncoder::Capabilities(/*loss_notification=*/false)};
9146 encoder_settings.encoder_factory = &encoder_factory;
9147 encoder_settings.bitrate_allocator_factory = bitrate_allocator_factory.get();
9148
9149 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9150 EXPECT_CALL((*adapter.get()), Initialize).WillOnce(Return());
9151
9152 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
9153 encoder_queue(new SuperLazyTaskQueue());
9154
9155 // Construct a VideoStreamEncoder instance and let it go out of scope without
9156 // doing anything else (including calling Stop()). This should be fine since
9157 // the posted init task will simply be deleted.
9158 auto encoder = std::make_unique<VideoStreamEncoder>(
9159 time_controller.GetClock(), 1, stats_proxy.get(), encoder_settings,
Markus Handell8e4197b2022-05-30 15:45:28 +02009160 std::make_unique<CpuOveruseDetectorProxy>(stats_proxy.get(),
9161 field_trials),
Tommi62b01db2022-01-25 23:41:22 +01009162 std::move(adapter), std::move(encoder_queue),
9163 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009164 kVideoBitrateAllocation,
9165 field_trials);
Tommi4d6dd172022-05-23 09:34:41 +02009166
9167 // Stop the encoder explicitly. This additional step tests if we could
9168 // hang when calling stop and the TQ has been stopped and/or isn't accepting
9169 // any more tasks.
9170 encoder->Stop();
Tommi62b01db2022-01-25 23:41:22 +01009171}
9172
Markus Handellb4e96d42021-11-05 12:00:55 +01009173TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
9174 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9175 auto* adapter_ptr = adapter.get();
9176 SimpleVideoStreamEncoderFactory factory;
Markus Handell8d87c462021-12-16 11:37:16 +01009177 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9178 nullptr;
9179 EXPECT_CALL(*adapter_ptr, Initialize)
9180 .WillOnce(Invoke([&video_stream_encoder_callback](
9181 FrameCadenceAdapterInterface::Callback* callback) {
9182 video_stream_encoder_callback = callback;
9183 }));
9184 TaskQueueBase* encoder_queue = nullptr;
9185 auto video_stream_encoder =
9186 factory.Create(std::move(adapter), &encoder_queue);
Markus Handellb4e96d42021-11-05 12:00:55 +01009187
Markus Handelle59fee82021-12-23 09:29:23 +01009188 // First a call before we know the frame size and hence cannot compute the
9189 // number of simulcast layers.
9190 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
9191 &FrameCadenceAdapterInterface::
9192 ZeroHertzModeParams::num_simulcast_layers,
Markus Handell5a77e512022-09-01 12:51:50 +00009193 Eq(0u)))));
Markus Handellb4e96d42021-11-05 12:00:55 +01009194 VideoEncoderConfig config;
Markus Handell8d87c462021-12-16 11:37:16 +01009195 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
Markus Handellb4e96d42021-11-05 12:00:55 +01009196 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9197 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
Markus Handelle59fee82021-12-23 09:29:23 +01009198 factory.DepleteTaskQueues();
9199
9200 // Then a call as we've computed the number of simulcast layers after a passed
9201 // frame.
9202 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
9203 &FrameCadenceAdapterInterface::
9204 ZeroHertzModeParams::num_simulcast_layers,
Markus Handell5a77e512022-09-01 12:51:50 +00009205 Gt(0u)))));
Markus Handell8d87c462021-12-16 11:37:16 +01009206 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handell9a478b52021-11-18 16:07:01 +01009207 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01009208 Mock::VerifyAndClearExpectations(adapter_ptr);
9209
Markus Handelle59fee82021-12-23 09:29:23 +01009210 // Expect a disabled zero-hertz mode after passing realtime video.
Markus Handell8d87c462021-12-16 11:37:16 +01009211 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Eq(absl::nullopt)));
Markus Handellb4e96d42021-11-05 12:00:55 +01009212 VideoEncoderConfig config2;
Markus Handell8d87c462021-12-16 11:37:16 +01009213 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config2);
Markus Handellb4e96d42021-11-05 12:00:55 +01009214 config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
9215 video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
Markus Handell8d87c462021-12-16 11:37:16 +01009216 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
Markus Handell9a478b52021-11-18 16:07:01 +01009217 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01009218}
9219
9220TEST(VideoStreamEncoderFrameCadenceTest,
9221 ForwardsFramesIntoFrameCadenceAdapter) {
9222 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9223 auto* adapter_ptr = adapter.get();
9224 test::FrameForwarder video_source;
9225 SimpleVideoStreamEncoderFactory factory;
9226 auto video_stream_encoder = factory.Create(std::move(adapter));
9227 video_stream_encoder->SetSource(
9228 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9229
9230 EXPECT_CALL(*adapter_ptr, OnFrame);
9231 auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
9232 video_source.IncomingCapturedFrame(
Markus Handell8d87c462021-12-16 11:37:16 +01009233 VideoFrame::Builder().set_video_frame_buffer(std::move(buffer)).build());
Markus Handellb4e96d42021-11-05 12:00:55 +01009234}
9235
Markus Handellee225432021-11-29 12:35:12 +01009236TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) {
9237 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9238 auto* adapter_ptr = adapter.get();
9239 test::FrameForwarder video_source;
9240 SimpleVideoStreamEncoderFactory factory;
9241 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9242 nullptr;
9243 EXPECT_CALL(*adapter_ptr, Initialize)
9244 .WillOnce(Invoke([&video_stream_encoder_callback](
9245 FrameCadenceAdapterInterface::Callback* callback) {
9246 video_stream_encoder_callback = callback;
9247 }));
9248 TaskQueueBase* encoder_queue = nullptr;
9249 auto video_stream_encoder =
9250 factory.Create(std::move(adapter), &encoder_queue);
9251
9252 // This is just to make the VSE operational. We'll feed a frame directly by
9253 // the callback interface.
9254 video_stream_encoder->SetSource(
9255 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9256
9257 VideoEncoderConfig video_encoder_config;
9258 test::FillEncoderConfiguration(kVideoCodecGeneric, 1, &video_encoder_config);
9259 video_stream_encoder->ConfigureEncoder(std::move(video_encoder_config),
9260 /*max_data_payload_length=*/1000);
9261
9262 EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps);
9263 EXPECT_CALL(*adapter_ptr, UpdateFrameRate);
Markus Handell8d87c462021-12-16 11:37:16 +01009264 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handellee225432021-11-29 12:35:12 +01009265 factory.DepleteTaskQueues();
9266}
9267
Markus Handell8d87c462021-12-16 11:37:16 +01009268TEST(VideoStreamEncoderFrameCadenceTest,
9269 DeactivatesActivatesLayersOnBitrateChanges) {
9270 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9271 auto* adapter_ptr = adapter.get();
9272 SimpleVideoStreamEncoderFactory factory;
9273 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9274 nullptr;
9275 EXPECT_CALL(*adapter_ptr, Initialize)
9276 .WillOnce(Invoke([&video_stream_encoder_callback](
9277 FrameCadenceAdapterInterface::Callback* callback) {
9278 video_stream_encoder_callback = callback;
9279 }));
9280 TaskQueueBase* encoder_queue = nullptr;
9281 auto video_stream_encoder =
9282 factory.Create(std::move(adapter), &encoder_queue);
9283
9284 // Configure 2 simulcast layers. FillEncoderConfiguration sets min bitrates to
9285 // {150000, 450000}.
9286 VideoEncoderConfig video_encoder_config;
9287 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9288 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9289 kMaxPayloadLength);
9290 // Ensure an encoder is created.
9291 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9292
9293 // Both layers enabled at 1 MBit/s.
9294 video_stream_encoder->OnBitrateUpdated(
9295 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9296 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9297 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9298 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9299 factory.DepleteTaskQueues();
9300 Mock::VerifyAndClearExpectations(adapter_ptr);
9301
9302 // Layer 1 disabled at 200 KBit/s.
9303 video_stream_encoder->OnBitrateUpdated(
9304 DataRate::KilobitsPerSec(200), DataRate::KilobitsPerSec(200),
9305 DataRate::KilobitsPerSec(200), 0, 0, 0);
9306 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9307 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9308 factory.DepleteTaskQueues();
9309 Mock::VerifyAndClearExpectations(adapter_ptr);
9310
9311 // All layers off at suspended video.
9312 video_stream_encoder->OnBitrateUpdated(DataRate::Zero(), DataRate::Zero(),
9313 DataRate::Zero(), 0, 0, 0);
9314 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/false));
9315 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9316 factory.DepleteTaskQueues();
9317 Mock::VerifyAndClearExpectations(adapter_ptr);
9318
9319 // Both layers enabled again back at 1 MBit/s.
9320 video_stream_encoder->OnBitrateUpdated(
9321 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9322 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9323 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9324 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9325 factory.DepleteTaskQueues();
9326}
9327
9328TEST(VideoStreamEncoderFrameCadenceTest, UpdatesQualityConvergence) {
9329 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9330 auto* adapter_ptr = adapter.get();
9331 SimpleVideoStreamEncoderFactory factory;
9332 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9333 nullptr;
9334 EXPECT_CALL(*adapter_ptr, Initialize)
9335 .WillOnce(Invoke([&video_stream_encoder_callback](
9336 FrameCadenceAdapterInterface::Callback* callback) {
9337 video_stream_encoder_callback = callback;
9338 }));
9339 TaskQueueBase* encoder_queue = nullptr;
9340 auto video_stream_encoder =
9341 factory.Create(std::move(adapter), &encoder_queue);
9342
9343 // Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder.
9344 VideoEncoderConfig video_encoder_config;
9345 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9346 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9347 kMaxPayloadLength);
9348 video_stream_encoder->OnBitrateUpdated(
9349 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9350 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9351
9352 // Pass a frame which has unconverged results.
9353 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9354 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9355 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9356 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
Markus Handelle1a198b2022-09-15 08:13:25 +00009357 encoded_image.qp_ = kVp8SteadyStateQpThreshold + 1;
Markus Handell8d87c462021-12-16 11:37:16 +01009358 CodecSpecificInfo codec_specific;
Markus Handelle1a198b2022-09-15 08:13:25 +00009359 codec_specific.codecType = kVideoCodecVP8;
Markus Handell8d87c462021-12-16 11:37:16 +01009360 return codec_specific;
9361 }));
9362 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, false));
9363 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9364 factory.DepleteTaskQueues();
9365 Mock::VerifyAndClearExpectations(adapter_ptr);
9366 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9367
9368 // Pass a frame which converges in layer 0 and not in layer 1.
9369 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9370 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9371 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9372 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
Markus Handelle1a198b2022-09-15 08:13:25 +00009373 // This sets spatial index 0 content to be at target quality, while
9374 // index 1 content is not.
9375 encoded_image.qp_ = kVp8SteadyStateQpThreshold +
9376 (encoded_image.SpatialIndex() == 0 ? 0 : 1);
Markus Handell8d87c462021-12-16 11:37:16 +01009377 CodecSpecificInfo codec_specific;
Markus Handelle1a198b2022-09-15 08:13:25 +00009378 codec_specific.codecType = kVideoCodecVP8;
Markus Handell8d87c462021-12-16 11:37:16 +01009379 return codec_specific;
9380 }));
9381 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, true));
9382 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9383 factory.DepleteTaskQueues();
9384 Mock::VerifyAndClearExpectations(adapter_ptr);
9385 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9386}
9387
Markus Handell2e0f4f02021-12-21 19:14:58 +01009388TEST(VideoStreamEncoderFrameCadenceTest,
9389 RequestsRefreshFramesWhenCadenceAdapterInstructs) {
9390 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9391 auto* adapter_ptr = adapter.get();
9392 MockVideoSourceInterface mock_source;
9393 SimpleVideoStreamEncoderFactory factory;
9394 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9395 nullptr;
9396 EXPECT_CALL(*adapter_ptr, Initialize)
9397 .WillOnce(Invoke([&video_stream_encoder_callback](
9398 FrameCadenceAdapterInterface::Callback* callback) {
9399 video_stream_encoder_callback = callback;
9400 }));
9401 TaskQueueBase* encoder_queue = nullptr;
9402 auto video_stream_encoder =
9403 factory.Create(std::move(adapter), &encoder_queue);
9404 video_stream_encoder->SetSource(
9405 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9406 VideoEncoderConfig config;
9407 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9408 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9409 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9410 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9411 // Ensure the encoder is set up.
9412 factory.DepleteTaskQueues();
9413
Markus Handell818e7fb2021-12-30 13:01:33 +01009414 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest)
9415 .WillOnce(Invoke([video_stream_encoder_callback] {
9416 video_stream_encoder_callback->RequestRefreshFrame();
9417 }));
Markus Handell2e0f4f02021-12-21 19:14:58 +01009418 EXPECT_CALL(mock_source, RequestRefreshFrame);
9419 video_stream_encoder->SendKeyFrame();
9420 factory.DepleteTaskQueues();
9421 Mock::VerifyAndClearExpectations(adapter_ptr);
9422 Mock::VerifyAndClearExpectations(&mock_source);
9423
Markus Handell818e7fb2021-12-30 13:01:33 +01009424 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest);
Markus Handell2e0f4f02021-12-21 19:14:58 +01009425 EXPECT_CALL(mock_source, RequestRefreshFrame).Times(0);
9426 video_stream_encoder->SendKeyFrame();
9427 factory.DepleteTaskQueues();
9428}
9429
Markus Handell818e7fb2021-12-30 13:01:33 +01009430TEST(VideoStreamEncoderFrameCadenceTest,
9431 RequestsRefreshFrameForEarlyZeroHertzKeyFrameRequest) {
9432 SimpleVideoStreamEncoderFactory factory;
9433 auto encoder_queue =
9434 factory.GetTimeController()->GetTaskQueueFactory()->CreateTaskQueue(
9435 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
9436
9437 // Enables zero-hertz mode.
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009438 test::ScopedKeyValueConfig field_trials(
9439 "WebRTC-ZeroHertzScreenshare/Enabled/");
Markus Handell818e7fb2021-12-30 13:01:33 +01009440 auto adapter = FrameCadenceAdapterInterface::Create(
Jonas Oreland8ca06132022-03-14 12:52:48 +01009441 factory.GetTimeController()->GetClock(), encoder_queue.get(),
9442 field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009443 FrameCadenceAdapterInterface* adapter_ptr = adapter.get();
9444
9445 MockVideoSourceInterface mock_source;
9446 auto video_stream_encoder = factory.CreateWithEncoderQueue(
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009447 std::move(adapter), std::move(encoder_queue), &field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009448
9449 video_stream_encoder->SetSource(
9450 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9451 VideoEncoderConfig config;
9452 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9453 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9454 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9455
9456 // Eventually expect a refresh frame request when requesting a key frame
9457 // before initializing zero-hertz mode. This can happen in reality because the
9458 // threads invoking key frame requests and constraints setup aren't
9459 // synchronized.
9460 EXPECT_CALL(mock_source, RequestRefreshFrame);
9461 video_stream_encoder->SendKeyFrame();
Markus Handellf5a50792022-06-16 14:25:15 +02009462 constexpr int kMaxFps = 30;
9463 adapter_ptr->OnConstraintsChanged(VideoTrackSourceConstraints{0, kMaxFps});
9464 factory.GetTimeController()->AdvanceTime(
9465 TimeDelta::Seconds(1) *
9466 FrameCadenceAdapterInterface::kOnDiscardedFrameRefreshFramePeriod /
9467 kMaxFps);
Markus Handell818e7fb2021-12-30 13:01:33 +01009468}
9469
perkj26091b12016-09-01 01:17:40 -07009470} // namespace webrtc