blob: 5271654ac9c6bb5037c7b0bb5b4ca92d3dd6801e [file] [log] [blame]
Jonas Oreland1262eb52022-09-27 16:53:04 +02001
perkj26091b12016-09-01 01:17:40 -07002/*
3 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
4 *
5 * Use of this source code is governed by a BSD-style license
6 * that can be found in the LICENSE file in the root of the source
7 * tree. An additional intellectual property rights grant can be found
8 * in the file PATENTS. All contributing project authors may
9 * be found in the AUTHORS file in the root of the source tree.
10 */
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Markus Handell8e4197b2022-05-30 15:45:28 +020020#include "api/field_trials_view.h"
Markus Handell9a478b52021-11-18 16:07:01 +010021#include "api/rtp_parameters.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020022#include "api/task_queue/default_task_queue_factory.h"
Markus Handell818e7fb2021-12-30 13:01:33 +010023#include "api/task_queue/task_queue_base.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010024#include "api/task_queue/task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020025#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010026#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010027#include "api/test/mock_video_encoder_factory.h"
Markus Handell8d87c462021-12-16 11:37:16 +010028#include "api/units/data_rate.h"
Markus Handellf5a50792022-06-16 14:25:15 +020029#include "api/units/time_delta.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080030#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020031#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020032#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020033#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010034#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010035#include "api/video_codecs/sdp_video_format.h"
Erik Språnge4589cb2022-04-06 16:44:30 +020036#include "api/video_codecs/video_codec.h"
Elad Alon370f93a2019-06-11 14:57:57 +020037#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020038#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010039#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020040#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010041#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020042#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070043#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080044#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020045#include "media/engine/webrtc_video_engine.h"
philipel09a28482022-05-25 09:47:06 +020046#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010047#include "modules/video_coding/codecs/h264/include/h264.h"
48#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
49#include "modules/video_coding/codecs/vp8/include/vp8.h"
50#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020051#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020052#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020053#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010054#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Markus Handelle1a198b2022-09-15 08:13:25 +000055#include "modules/video_coding/utility/vp8_constants.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020056#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010057#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020058#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020059#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080060#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020061#include "rtc_base/synchronization/mutex.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020062#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020063#include "test/encoder_settings.h"
64#include "test/fake_encoder.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010065#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020066#include "test/gmock.h"
67#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010068#include "test/mappable_native_buffer.h"
Jonas Orelandc7f691a2022-03-09 15:12:07 +010069#include "test/scoped_key_value_config.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020070#include "test/time_controller/simulated_time_controller.h"
Byoungchan Lee13fe3672022-04-06 10:44:42 +090071#include "test/video_encoder_nullable_proxy_factory.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020072#include "test/video_encoder_proxy_factory.h"
Jonas Oreland1262eb52022-09-27 16:53:04 +020073#include "video/config/encoder_stream_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010074#include "video/frame_cadence_adapter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020075#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070076
77namespace webrtc {
78
sprang57c2fff2017-01-16 06:24:02 -080079using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020080using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020081using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020082using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020083using ::testing::Ge;
84using ::testing::Gt;
Markus Handellee225432021-11-29 12:35:12 +010085using ::testing::Invoke;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020086using ::testing::Le;
87using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010088using ::testing::Matcher;
Markus Handellb4e96d42021-11-05 12:00:55 +010089using ::testing::Mock;
philipel9b058032020-02-10 11:30:00 +010090using ::testing::NiceMock;
Markus Handellb4e96d42021-11-05 12:00:55 +010091using ::testing::Optional;
philipel9b058032020-02-10 11:30:00 +010092using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010093using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020094using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080095
perkj803d97f2016-11-01 11:45:46 -070096namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020097const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010098const int kQpLow = 1;
99const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200100const int kMinFramerateFps = 2;
101const int kMinBalancedFramerateFps = 7;
Markus Handell2cfc1af2022-08-19 08:16:48 +0000102constexpr TimeDelta kFrameTimeout = TimeDelta::Millis(100);
asapersson5f7226f2016-11-25 04:37:00 -0800103const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 10:07:11 +0200104const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
105const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
106const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
107const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 07:02:22 -0800108const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -0700109const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200110const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +0200111const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200112const VideoEncoder::ResolutionBitrateLimits
113 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
114const VideoEncoder::ResolutionBitrateLimits
115 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800116
Asa Persson606d3cb2021-10-04 10:07:11 +0200117uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200118 0x00, 0x00, 0x03, 0x03, 0xF4,
119 0x05, 0x03, 0xC7, 0xE0, 0x1B,
120 0x41, 0x10, 0x8D, 0x00};
121
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100122const uint8_t kCodedFrameVp8Qp25[] = {
123 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
124 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
125 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
126
Markus Handell818e7fb2021-12-30 13:01:33 +0100127VideoFrame CreateSimpleNV12Frame() {
128 return VideoFrame::Builder()
129 .set_video_frame_buffer(rtc::make_ref_counted<NV12Buffer>(
130 /*width=*/16, /*height=*/16))
131 .build();
132}
133
Markus Handell8d87c462021-12-16 11:37:16 +0100134void PassAFrame(
135 TaskQueueBase* encoder_queue,
136 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback,
137 int64_t ntp_time_ms) {
Danil Chapovalov95eeaa72022-07-06 10:14:29 +0200138 encoder_queue->PostTask([video_stream_encoder_callback, ntp_time_ms] {
139 video_stream_encoder_callback->OnFrame(Timestamp::Millis(ntp_time_ms), 1,
140 CreateSimpleNV12Frame());
141 });
Markus Handell8d87c462021-12-16 11:37:16 +0100142}
143
perkj803d97f2016-11-01 11:45:46 -0700144class TestBuffer : public webrtc::I420Buffer {
145 public:
146 TestBuffer(rtc::Event* event, int width, int height)
147 : I420Buffer(width, height), event_(event) {}
148
149 private:
150 friend class rtc::RefCountedObject<TestBuffer>;
151 ~TestBuffer() override {
152 if (event_)
153 event_->Set();
154 }
155 rtc::Event* const event_;
156};
157
Henrik Boström56db9ff2021-03-24 09:06:45 +0100158// A fake native buffer that can't be converted to I420. Upon scaling, it
159// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700160class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
161 public:
162 FakeNativeBuffer(rtc::Event* event, int width, int height)
163 : event_(event), width_(width), height_(height) {}
164 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
165 int width() const override { return width_; }
166 int height() const override { return height_; }
167 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
168 return nullptr;
169 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100170 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
171 int offset_x,
172 int offset_y,
173 int crop_width,
174 int crop_height,
175 int scaled_width,
176 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200177 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
178 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100179 }
Noah Richards51db4212019-06-12 06:59:12 -0700180
181 private:
182 friend class rtc::RefCountedObject<FakeNativeBuffer>;
183 ~FakeNativeBuffer() override {
184 if (event_)
185 event_->Set();
186 }
187 rtc::Event* const event_;
188 const int width_;
189 const int height_;
190};
191
Evan Shrubsole895556e2020-10-05 09:15:13 +0200192// A fake native buffer that is backed by an NV12 buffer.
193class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
194 public:
195 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
196 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
197
198 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
199 int width() const override { return nv12_buffer_->width(); }
200 int height() const override { return nv12_buffer_->height(); }
201 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
202 return nv12_buffer_->ToI420();
203 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200204 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
205 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
206 if (absl::c_find(types, Type::kNV12) != types.end()) {
207 return nv12_buffer_;
208 }
209 return nullptr;
210 }
Niels Möllerba2de582022-04-20 16:46:26 +0200211 const NV12BufferInterface* GetNV12() const { return nv12_buffer_.get(); }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200212
213 private:
214 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
215 ~FakeNV12NativeBuffer() override {
216 if (event_)
217 event_->Set();
218 }
219 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
220 rtc::Event* const event_;
221};
222
Niels Möller7dc26b72017-12-06 10:27:48 +0100223class CpuOveruseDetectorProxy : public OveruseFrameDetector {
224 public:
Markus Handell8e4197b2022-05-30 15:45:28 +0200225 CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer,
226 const FieldTrialsView& field_trials)
227 : OveruseFrameDetector(metrics_observer, field_trials),
Henrik Boström381d1092020-05-12 18:49:07 +0200228 last_target_framerate_fps_(-1),
229 framerate_updated_event_(true /* manual_reset */,
230 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100231 virtual ~CpuOveruseDetectorProxy() {}
232
233 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200234 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100235 last_target_framerate_fps_ = framerate_fps;
236 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200237 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100238 }
239
240 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200241 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100242 return last_target_framerate_fps_;
243 }
244
Niels Möller4db138e2018-04-19 09:04:13 +0200245 CpuOveruseOptions GetOptions() { return options_; }
246
Henrik Boström381d1092020-05-12 18:49:07 +0200247 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
248
Niels Möller7dc26b72017-12-06 10:27:48 +0100249 private:
Markus Handella3765182020-07-08 13:13:32 +0200250 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100251 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200252 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100253};
254
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200255class FakeVideoSourceRestrictionsListener
256 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200257 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200258 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200259 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200260 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200261 RTC_DCHECK(was_restrictions_updated_);
262 }
263
264 rtc::Event* restrictions_updated_event() {
265 return &restrictions_updated_event_;
266 }
267
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200268 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200269 void OnVideoSourceRestrictionsUpdated(
270 VideoSourceRestrictions restrictions,
271 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200272 rtc::scoped_refptr<Resource> reason,
273 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200274 was_restrictions_updated_ = true;
275 restrictions_updated_event_.Set();
276 }
277
278 private:
279 bool was_restrictions_updated_;
280 rtc::Event restrictions_updated_event_;
281};
282
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200283auto WantsFps(Matcher<int> fps_matcher) {
284 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
285 fps_matcher);
286}
287
288auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
289 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
290 AllOf(max_pixel_matcher, Gt(0)));
291}
292
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200293auto ResolutionMax() {
294 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200295 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200296 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
297 Eq(absl::nullopt)));
298}
299
300auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200301 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200302}
303
304auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200305 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200306}
307
308auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200309 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200310}
311
312auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200313 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200314}
315
316auto FpsMaxResolutionMax() {
317 return AllOf(FpsMax(), ResolutionMax());
318}
319
320auto UnlimitedSinkWants() {
321 return AllOf(FpsUnlimited(), ResolutionMax());
322}
323
324auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
325 Matcher<int> fps_range_matcher;
326
327 if (last_frame_pixels <= 320 * 240) {
328 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200329 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200330 fps_range_matcher = AllOf(Ge(10), Le(15));
331 } else if (last_frame_pixels <= 640 * 480) {
332 fps_range_matcher = Ge(15);
333 } else {
334 fps_range_matcher = Eq(kDefaultFramerate);
335 }
336 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
337 fps_range_matcher);
338}
339
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200340auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
341 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
342 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
343}
344
345auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
346 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
347}
348
349auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
350 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
351}
352
353auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
354 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
355 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
356}
357
358auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
359 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
360 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
361}
362
363auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
364 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
365 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
366}
367
368auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
369 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
370 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
371}
372
mflodmancc3d4422017-08-03 08:27:51 -0700373class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700374 public:
Markus Handell9a478b52021-11-18 16:07:01 +0100375 VideoStreamEncoderUnderTest(
376 TimeController* time_controller,
377 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter,
378 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
379 encoder_queue,
380 SendStatisticsProxy* stats_proxy,
381 const VideoStreamEncoderSettings& settings,
382 VideoStreamEncoder::BitrateAllocationCallbackType
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100383 allocation_callback_type,
Erik Språnge4589cb2022-04-06 16:44:30 +0200384 const FieldTrialsView& field_trials,
385 int num_cores)
Markus Handell8e4197b2022-05-30 15:45:28 +0200386 : VideoStreamEncoder(
387 time_controller->GetClock(),
388 num_cores,
389 stats_proxy,
390 settings,
391 std::unique_ptr<OveruseFrameDetector>(
392 overuse_detector_proxy_ =
393 new CpuOveruseDetectorProxy(stats_proxy, field_trials)),
394 std::move(cadence_adapter),
395 std::move(encoder_queue),
396 allocation_callback_type,
397 field_trials),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200398 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200399 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200400 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200401 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200402 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200403 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200404 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200405 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100406 }
perkj803d97f2016-11-01 11:45:46 -0700407
Henrik Boström381d1092020-05-12 18:49:07 +0200408 void SetSourceAndWaitForRestrictionsUpdated(
409 rtc::VideoSourceInterface<VideoFrame>* source,
410 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200411 FakeVideoSourceRestrictionsListener listener;
412 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200413 SetSource(source, degradation_preference);
Markus Handell2cfc1af2022-08-19 08:16:48 +0000414 listener.restrictions_updated_event()->Wait(TimeDelta::Seconds(5));
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200415 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200416 }
417
418 void SetSourceAndWaitForFramerateUpdated(
419 rtc::VideoSourceInterface<VideoFrame>* source,
420 const DegradationPreference& degradation_preference) {
421 overuse_detector_proxy_->framerate_updated_event()->Reset();
422 SetSource(source, degradation_preference);
Markus Handell2cfc1af2022-08-19 08:16:48 +0000423 overuse_detector_proxy_->framerate_updated_event()->Wait(
424 TimeDelta::Seconds(5));
Henrik Boström381d1092020-05-12 18:49:07 +0200425 }
426
427 void OnBitrateUpdatedAndWaitForManagedResources(
428 DataRate target_bitrate,
429 DataRate stable_target_bitrate,
430 DataRate link_allocation,
431 uint8_t fraction_lost,
432 int64_t round_trip_time_ms,
433 double cwnd_reduce_ratio) {
434 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
435 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
436 // Bitrate is updated on the encoder queue.
437 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200438 }
439
kthelgason2fc52542017-03-03 00:24:41 -0800440 // This is used as a synchronisation mechanism, to make sure that the
441 // encoder queue is not blocked before we start sending it frames.
442 void WaitUntilTaskQueueIsIdle() {
Markus Handell28c71802021-11-08 10:11:55 +0100443 time_controller_->AdvanceTime(TimeDelta::Zero());
kthelgason2fc52542017-03-03 00:24:41 -0800444 }
445
Henrik Boström91aa7322020-04-28 12:24:33 +0200446 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200447 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200448 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200449 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200450 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200451 event.Set();
452 });
Markus Handell2cfc1af2022-08-19 08:16:48 +0000453 ASSERT_TRUE(event.Wait(TimeDelta::Seconds(5)));
Markus Handell28c71802021-11-08 10:11:55 +0100454 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200455 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200456
Henrik Boström91aa7322020-04-28 12:24:33 +0200457 void TriggerCpuUnderuse() {
458 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200459 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200460 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200461 event.Set();
462 });
Markus Handell2cfc1af2022-08-19 08:16:48 +0000463 ASSERT_TRUE(event.Wait(TimeDelta::Seconds(5)));
Markus Handell28c71802021-11-08 10:11:55 +0100464 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200465 }
kthelgason876222f2016-11-29 01:44:11 -0800466
Henrik Boström91aa7322020-04-28 12:24:33 +0200467 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200468 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200469 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200470 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200471 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200472 event.Set();
473 });
Markus Handell2cfc1af2022-08-19 08:16:48 +0000474 ASSERT_TRUE(event.Wait(TimeDelta::Seconds(5)));
Markus Handell28c71802021-11-08 10:11:55 +0100475 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200476 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200477 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200478 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200479 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200480 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200481 event.Set();
482 });
Markus Handell2cfc1af2022-08-19 08:16:48 +0000483 ASSERT_TRUE(event.Wait(TimeDelta::Seconds(5)));
Markus Handell28c71802021-11-08 10:11:55 +0100484 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200485 }
486
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200487 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100488 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200489 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
490 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200491 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700492};
493
Noah Richards51db4212019-06-12 06:59:12 -0700494// Simulates simulcast behavior and makes highest stream resolutions divisible
495// by 4.
496class CroppingVideoStreamFactory
497 : public VideoEncoderConfig::VideoStreamFactoryInterface {
498 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200499 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700500
501 private:
502 std::vector<VideoStream> CreateEncoderStreams(
Jonas Oreland80c87d72022-09-29 15:01:09 +0200503 int frame_width,
504 int frame_height,
Noah Richards51db4212019-06-12 06:59:12 -0700505 const VideoEncoderConfig& encoder_config) override {
506 std::vector<VideoStream> streams = test::CreateVideoStreams(
Jonas Oreland80c87d72022-09-29 15:01:09 +0200507 frame_width - frame_width % 4, frame_height - frame_height % 4,
508 encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700509 return streams;
510 }
Noah Richards51db4212019-06-12 06:59:12 -0700511};
512
sprangb1ca0732017-02-01 08:38:12 -0800513class AdaptingFrameForwarder : public test::FrameForwarder {
514 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200515 explicit AdaptingFrameForwarder(TimeController* time_controller)
516 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700517 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800518
519 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200520 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800521 adaptation_enabled_ = enabled;
522 }
523
asaperssonfab67072017-04-04 05:51:49 -0700524 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200525 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800526 return adaptation_enabled_;
527 }
528
Henrik Boström1124ed12021-02-25 10:30:39 +0100529 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
530 // the resolution or frame rate was different than it is currently. If
531 // something else is modified, such as encoder resolutions, but the resolution
532 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700533 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200534 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700535 return last_wants_;
536 }
537
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200538 absl::optional<int> last_sent_width() const { return last_width_; }
539 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800540
sprangb1ca0732017-02-01 08:38:12 -0800541 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200542 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +0100543 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200544
sprangb1ca0732017-02-01 08:38:12 -0800545 int cropped_width = 0;
546 int cropped_height = 0;
547 int out_width = 0;
548 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700549 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000550 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
551 << "w=" << video_frame.width()
552 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700553 if (adapter_.AdaptFrameResolution(
554 video_frame.width(), video_frame.height(),
555 video_frame.timestamp_us() * 1000, &cropped_width,
556 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100557 VideoFrame adapted_frame =
558 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200559 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100560 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200561 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100562 .set_timestamp_ms(99)
563 .set_rotation(kVideoRotation_0)
564 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100565 if (video_frame.has_update_rect()) {
566 adapted_frame.set_update_rect(
567 video_frame.update_rect().ScaleWithFrame(
568 video_frame.width(), video_frame.height(), 0, 0,
569 video_frame.width(), video_frame.height(), out_width,
570 out_height));
571 }
sprangc5d62e22017-04-02 23:53:04 -0700572 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800573 last_width_.emplace(adapted_frame.width());
574 last_height_.emplace(adapted_frame.height());
575 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200576 last_width_ = absl::nullopt;
577 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700578 }
sprangb1ca0732017-02-01 08:38:12 -0800579 } else {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000580 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800581 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800582 last_width_.emplace(video_frame.width());
583 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800584 }
585 }
586
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200587 void OnOutputFormatRequest(int width, int height) {
588 absl::optional<std::pair<int, int>> target_aspect_ratio =
589 std::make_pair(width, height);
590 absl::optional<int> max_pixel_count = width * height;
591 absl::optional<int> max_fps;
592 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
593 max_fps);
594 }
595
sprangb1ca0732017-02-01 08:38:12 -0800596 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
597 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200598 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100599 rtc::VideoSinkWants prev_wants = sink_wants_locked();
600 bool did_adapt =
601 prev_wants.max_pixel_count != wants.max_pixel_count ||
602 prev_wants.target_pixel_count != wants.target_pixel_count ||
603 prev_wants.max_framerate_fps != wants.max_framerate_fps;
604 if (did_adapt) {
605 last_wants_ = prev_wants;
606 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100607 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200608 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800609 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200610
Erik Språng5e13d052022-08-02 11:42:49 +0200611 void RequestRefreshFrame() override { ++refresh_frames_requested_; }
612
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200613 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800614 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200615 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
616 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200617 absl::optional<int> last_width_;
618 absl::optional<int> last_height_;
Erik Språng5e13d052022-08-02 11:42:49 +0200619 int refresh_frames_requested_{0};
sprangb1ca0732017-02-01 08:38:12 -0800620};
sprangc5d62e22017-04-02 23:53:04 -0700621
Niels Möller213618e2018-07-24 09:29:58 +0200622// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700623class MockableSendStatisticsProxy : public SendStatisticsProxy {
624 public:
625 MockableSendStatisticsProxy(Clock* clock,
626 const VideoSendStream::Config& config,
Jonas Oreland8ca06132022-03-14 12:52:48 +0100627 VideoEncoderConfig::ContentType content_type,
Jonas Orelande62c2f22022-03-29 11:04:48 +0200628 const FieldTrialsView& field_trials)
Jonas Oreland8ca06132022-03-14 12:52:48 +0100629 : SendStatisticsProxy(clock, config, content_type, field_trials) {}
sprangc5d62e22017-04-02 23:53:04 -0700630
631 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200632 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700633 if (mock_stats_)
634 return *mock_stats_;
635 return SendStatisticsProxy::GetStats();
636 }
637
Niels Möller213618e2018-07-24 09:29:58 +0200638 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200639 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200640 if (mock_stats_)
641 return mock_stats_->input_frame_rate;
642 return SendStatisticsProxy::GetInputFrameRate();
643 }
sprangc5d62e22017-04-02 23:53:04 -0700644 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200645 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700646 mock_stats_.emplace(stats);
647 }
648
649 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200650 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700651 mock_stats_.reset();
652 }
653
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200654 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
655 on_frame_dropped_ = std::move(callback);
656 }
657
sprangc5d62e22017-04-02 23:53:04 -0700658 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200659 void OnFrameDropped(DropReason reason) override {
660 SendStatisticsProxy::OnFrameDropped(reason);
661 if (on_frame_dropped_)
662 on_frame_dropped_(reason);
663 }
664
Markus Handella3765182020-07-08 13:13:32 +0200665 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200666 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200667 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700668};
669
Markus Handellb4e96d42021-11-05 12:00:55 +0100670class SimpleVideoStreamEncoderFactory {
671 public:
672 class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
673 public:
674 using VideoStreamEncoder::VideoStreamEncoder;
675 ~AdaptedVideoStreamEncoder() { Stop(); }
676 };
677
Markus Handell8d87c462021-12-16 11:37:16 +0100678 class MockFakeEncoder : public test::FakeEncoder {
679 public:
680 using FakeEncoder::FakeEncoder;
681 MOCK_METHOD(CodecSpecificInfo,
682 EncodeHook,
683 (EncodedImage & encoded_image,
684 rtc::scoped_refptr<EncodedImageBuffer> buffer),
685 (override));
686 };
687
Markus Handellee225432021-11-29 12:35:12 +0100688 SimpleVideoStreamEncoderFactory() {
Markus Handellb4e96d42021-11-05 12:00:55 +0100689 encoder_settings_.encoder_factory = &encoder_factory_;
Markus Handellee225432021-11-29 12:35:12 +0100690 encoder_settings_.bitrate_allocator_factory =
691 bitrate_allocator_factory_.get();
Markus Handellb4e96d42021-11-05 12:00:55 +0100692 }
693
Markus Handell818e7fb2021-12-30 13:01:33 +0100694 std::unique_ptr<AdaptedVideoStreamEncoder> CreateWithEncoderQueue(
Markus Handellee225432021-11-29 12:35:12 +0100695 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100696 std::unique_ptr<TaskQueueBase, TaskQueueDeleter> encoder_queue,
Jonas Orelande62c2f22022-03-29 11:04:48 +0200697 const FieldTrialsView* field_trials = nullptr) {
Markus Handellb4e96d42021-11-05 12:00:55 +0100698 auto result = std::make_unique<AdaptedVideoStreamEncoder>(
699 time_controller_.GetClock(),
700 /*number_of_cores=*/1,
701 /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
Markus Handell8e4197b2022-05-30 15:45:28 +0200702 std::make_unique<CpuOveruseDetectorProxy>(
703 /*stats_proxy=*/nullptr,
704 field_trials ? *field_trials : field_trials_),
Markus Handellee225432021-11-29 12:35:12 +0100705 std::move(zero_hertz_adapter), std::move(encoder_queue),
Markus Handellb4e96d42021-11-05 12:00:55 +0100706 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100707 kVideoBitrateAllocation,
708 field_trials ? *field_trials : field_trials_);
Markus Handellb4e96d42021-11-05 12:00:55 +0100709 result->SetSink(&sink_, /*rotation_applied=*/false);
710 return result;
711 }
712
Markus Handell818e7fb2021-12-30 13:01:33 +0100713 std::unique_ptr<AdaptedVideoStreamEncoder> Create(
714 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
715 TaskQueueBase** encoder_queue_ptr = nullptr) {
716 auto encoder_queue =
717 time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
718 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
719 if (encoder_queue_ptr)
720 *encoder_queue_ptr = encoder_queue.get();
721 return CreateWithEncoderQueue(std::move(zero_hertz_adapter),
722 std::move(encoder_queue));
723 }
724
Markus Handell9a478b52021-11-18 16:07:01 +0100725 void DepleteTaskQueues() { time_controller_.AdvanceTime(TimeDelta::Zero()); }
Markus Handell8d87c462021-12-16 11:37:16 +0100726 MockFakeEncoder& GetMockFakeEncoder() { return mock_fake_encoder_; }
Markus Handell9a478b52021-11-18 16:07:01 +0100727
Markus Handell818e7fb2021-12-30 13:01:33 +0100728 GlobalSimulatedTimeController* GetTimeController() {
729 return &time_controller_;
730 }
731
Markus Handellb4e96d42021-11-05 12:00:55 +0100732 private:
733 class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
734 public:
735 ~NullEncoderSink() override = default;
736 void OnEncoderConfigurationChanged(
737 std::vector<VideoStream> streams,
738 bool is_svc,
739 VideoEncoderConfig::ContentType content_type,
740 int min_transmit_bitrate_bps) override {}
741 void OnBitrateAllocationUpdated(
742 const VideoBitrateAllocation& allocation) override {}
743 void OnVideoLayersAllocationUpdated(
744 VideoLayersAllocation allocation) override {}
745 Result OnEncodedImage(
746 const EncodedImage& encoded_image,
747 const CodecSpecificInfo* codec_specific_info) override {
748 return Result(EncodedImageCallback::Result::OK);
749 }
750 };
751
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100752 test::ScopedKeyValueConfig field_trials_;
Philipp Hanckea204ad22022-07-08 18:43:25 +0200753 GlobalSimulatedTimeController time_controller_{Timestamp::Zero()};
Markus Handellee225432021-11-29 12:35:12 +0100754 std::unique_ptr<TaskQueueFactory> task_queue_factory_{
755 time_controller_.CreateTaskQueueFactory()};
756 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_ =
757 std::make_unique<MockableSendStatisticsProxy>(
758 time_controller_.GetClock(),
759 VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 12:52:48 +0100760 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
761 field_trials_);
Markus Handellee225432021-11-29 12:35:12 +0100762 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_ =
763 CreateBuiltinVideoBitrateAllocatorFactory();
764 VideoStreamEncoderSettings encoder_settings_{
765 VideoEncoder::Capabilities(/*loss_notification=*/false)};
Markus Handell8d87c462021-12-16 11:37:16 +0100766 MockFakeEncoder mock_fake_encoder_{time_controller_.GetClock()};
767 test::VideoEncoderProxyFactory encoder_factory_{&mock_fake_encoder_};
Markus Handellb4e96d42021-11-05 12:00:55 +0100768 NullEncoderSink sink_;
769};
770
771class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
772 public:
773 MOCK_METHOD(void, Initialize, (Callback * callback), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100774 MOCK_METHOD(void,
775 SetZeroHertzModeEnabled,
776 (absl::optional<ZeroHertzModeParams>),
777 (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100778 MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
Markus Handellee225432021-11-29 12:35:12 +0100779 MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override));
780 MOCK_METHOD(void, UpdateFrameRate, (), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100781 MOCK_METHOD(void,
782 UpdateLayerQualityConvergence,
Markus Handell5a77e512022-09-01 12:51:50 +0000783 (size_t spatial_index, bool converged),
Markus Handell8d87c462021-12-16 11:37:16 +0100784 (override));
785 MOCK_METHOD(void,
786 UpdateLayerStatus,
Markus Handell5a77e512022-09-01 12:51:50 +0000787 (size_t spatial_index, bool enabled),
Markus Handell8d87c462021-12-16 11:37:16 +0100788 (override));
Markus Handell818e7fb2021-12-30 13:01:33 +0100789 MOCK_METHOD(void, ProcessKeyFrameRequest, (), (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100790};
791
philipel9b058032020-02-10 11:30:00 +0100792class MockEncoderSelector
793 : public VideoEncoderFactory::EncoderSelectorInterface {
794 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200795 MOCK_METHOD(void,
796 OnCurrentEncoder,
797 (const SdpVideoFormat& format),
798 (override));
799 MOCK_METHOD(absl::optional<SdpVideoFormat>,
800 OnAvailableBitrate,
801 (const DataRate& rate),
802 (override));
philipel6daa3042022-04-11 10:48:28 +0200803 MOCK_METHOD(absl::optional<SdpVideoFormat>,
804 OnResolutionChange,
805 (const RenderResolution& resolution),
806 (override));
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200807 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100808};
809
Markus Handell2e0f4f02021-12-21 19:14:58 +0100810class MockVideoSourceInterface : public rtc::VideoSourceInterface<VideoFrame> {
811 public:
812 MOCK_METHOD(void,
813 AddOrUpdateSink,
814 (rtc::VideoSinkInterface<VideoFrame>*,
815 const rtc::VideoSinkWants&),
816 (override));
817 MOCK_METHOD(void,
818 RemoveSink,
819 (rtc::VideoSinkInterface<VideoFrame>*),
820 (override));
821 MOCK_METHOD(void, RequestRefreshFrame, (), (override));
822};
823
perkj803d97f2016-11-01 11:45:46 -0700824} // namespace
825
mflodmancc3d4422017-08-03 08:27:51 -0700826class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700827 public:
Markus Handell2cfc1af2022-08-19 08:16:48 +0000828 static constexpr TimeDelta kDefaultTimeout = TimeDelta::Seconds(1);
perkj26091b12016-09-01 01:17:40 -0700829
mflodmancc3d4422017-08-03 08:27:51 -0700830 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700831 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700832 codec_width_(320),
833 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200834 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200835 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200836 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700837 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200838 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700839 video_send_config_,
Jonas Oreland8ca06132022-03-14 12:52:48 +0100840 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
841 field_trials_)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200842 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700843
844 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700845 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700846 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200847 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800848 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200849 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200850 video_send_config_.rtp.payload_name = "FAKE";
851 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700852
Per512ecb32016-09-23 15:52:06 +0200853 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200854 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200855 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
856 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
857 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100858 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700859
Niels Möllerf1338562018-04-26 09:51:47 +0200860 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800861 }
862
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100863 void ConfigureEncoder(
864 VideoEncoderConfig video_encoder_config,
865 VideoStreamEncoder::BitrateAllocationCallbackType
866 allocation_callback_type =
867 VideoStreamEncoder::BitrateAllocationCallbackType::
Erik Språnge4589cb2022-04-06 16:44:30 +0200868 kVideoBitrateAllocationWhenScreenSharing,
869 int num_cores = 1) {
mflodmancc3d4422017-08-03 08:27:51 -0700870 if (video_stream_encoder_)
871 video_stream_encoder_->Stop();
Markus Handell9a478b52021-11-18 16:07:01 +0100872
873 auto encoder_queue = GetTaskQueueFactory()->CreateTaskQueue(
874 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
875 TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
876 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter =
877 FrameCadenceAdapterInterface::Create(time_controller_.GetClock(),
Jonas Oreland8ca06132022-03-14 12:52:48 +0100878 encoder_queue_ptr, field_trials_);
Markus Handell9a478b52021-11-18 16:07:01 +0100879 video_stream_encoder_ = std::make_unique<VideoStreamEncoderUnderTest>(
880 &time_controller_, std::move(cadence_adapter), std::move(encoder_queue),
881 stats_proxy_.get(), video_send_config_.encoder_settings,
Erik Språnge4589cb2022-04-06 16:44:30 +0200882 allocation_callback_type, field_trials_, num_cores);
Asa Persson606d3cb2021-10-04 10:07:11 +0200883 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 08:27:51 -0700884 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700885 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 10:07:11 +0200886 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 08:27:51 -0700887 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200888 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700889 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800890 }
891
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100892 void ResetEncoder(const std::string& payload_name,
893 size_t num_streams,
894 size_t num_temporal_layers,
895 unsigned char num_spatial_layers,
896 bool screenshare,
897 VideoStreamEncoder::BitrateAllocationCallbackType
898 allocation_callback_type =
899 VideoStreamEncoder::BitrateAllocationCallbackType::
Erik Språnge4589cb2022-04-06 16:44:30 +0200900 kVideoBitrateAllocationWhenScreenSharing,
901 int num_cores = 1) {
Niels Möller259a4972018-04-05 15:36:51 +0200902 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800903
904 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200905 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
906 num_streams, &video_encoder_config);
907 for (auto& layer : video_encoder_config.simulcast_layers) {
908 layer.num_temporal_layers = num_temporal_layers;
909 layer.max_framerate = kDefaultFramerate;
910 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100911 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 10:07:11 +0200912 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 07:06:52 -0700913 video_encoder_config.content_type =
914 screenshare ? VideoEncoderConfig::ContentType::kScreen
915 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700916 if (payload_name == "VP9") {
917 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
918 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200919 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700920 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200921 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
922 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700923 }
Erik Språnge4589cb2022-04-06 16:44:30 +0200924 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type,
925 num_cores);
perkj26091b12016-09-01 01:17:40 -0700926 }
927
sprang57c2fff2017-01-16 06:24:02 -0800928 VideoFrame CreateFrame(int64_t ntp_time_ms,
929 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200930 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200931 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200932 destruction_event, codec_width_, codec_height_))
933 .set_ntp_time_ms(ntp_time_ms)
934 .set_timestamp_ms(99)
935 .set_rotation(kVideoRotation_0)
936 .build();
perkj26091b12016-09-01 01:17:40 -0700937 }
938
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100939 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
940 rtc::Event* destruction_event,
941 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200942 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200943 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200944 destruction_event, codec_width_, codec_height_))
945 .set_ntp_time_ms(ntp_time_ms)
946 .set_timestamp_ms(99)
947 .set_rotation(kVideoRotation_0)
948 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
949 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100950 }
951
sprang57c2fff2017-01-16 06:24:02 -0800952 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200953 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
954 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200955 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200956 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200957 .set_ntp_time_ms(ntp_time_ms)
958 .set_timestamp_ms(ntp_time_ms)
959 .set_rotation(kVideoRotation_0)
960 .build();
perkj803d97f2016-11-01 11:45:46 -0700961 }
962
Evan Shrubsole895556e2020-10-05 09:15:13 +0200963 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200964 return VideoFrame::Builder()
965 .set_video_frame_buffer(NV12Buffer::Create(width, height))
966 .set_ntp_time_ms(ntp_time_ms)
967 .set_timestamp_ms(ntp_time_ms)
968 .set_rotation(kVideoRotation_0)
969 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200970 }
971
Noah Richards51db4212019-06-12 06:59:12 -0700972 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
973 rtc::Event* destruction_event,
974 int width,
975 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200976 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200977 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200978 destruction_event, width, height))
979 .set_ntp_time_ms(ntp_time_ms)
980 .set_timestamp_ms(99)
981 .set_rotation(kVideoRotation_0)
982 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700983 }
984
Evan Shrubsole895556e2020-10-05 09:15:13 +0200985 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
986 rtc::Event* destruction_event,
987 int width,
988 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200989 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200990 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200991 destruction_event, width, height))
992 .set_ntp_time_ms(ntp_time_ms)
993 .set_timestamp_ms(99)
994 .set_rotation(kVideoRotation_0)
995 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200996 }
997
Noah Richards51db4212019-06-12 06:59:12 -0700998 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
999 rtc::Event* destruction_event) const {
1000 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
1001 codec_height_);
1002 }
1003
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001004 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02001005 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001006 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001007
1008 video_source_.IncomingCapturedFrame(
1009 CreateFrame(1, codec_width_, codec_height_));
1010 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +02001011 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001012 }
1013
sprang4847ae62017-06-27 07:06:52 -07001014 void WaitForEncodedFrame(int64_t expected_ntp_time) {
1015 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001016 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001017 }
1018
Markus Handell2cfc1af2022-08-19 08:16:48 +00001019 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, TimeDelta timeout) {
1020 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001021 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001022 return ok;
1023 }
1024
1025 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
1026 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001027 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001028 }
1029
1030 void ExpectDroppedFrame() {
1031 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001032 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001033 }
1034
Markus Handell2cfc1af2022-08-19 08:16:48 +00001035 bool WaitForFrame(TimeDelta timeout) {
1036 bool ok = sink_.WaitForFrame(timeout);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001037 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001038 return ok;
1039 }
1040
perkj26091b12016-09-01 01:17:40 -07001041 class TestEncoder : public test::FakeEncoder {
1042 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001043 explicit TestEncoder(TimeController* time_controller)
1044 : FakeEncoder(time_controller->GetClock()),
1045 time_controller_(time_controller) {
1046 RTC_DCHECK(time_controller_);
1047 }
perkj26091b12016-09-01 01:17:40 -07001048
Erik Språngaed30702018-11-05 12:57:17 +01001049 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +02001050 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +02001051 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +01001052 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001053 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +01001054 info.scaling_settings = VideoEncoder::ScalingSettings(
1055 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001056 }
1057 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001058 for (int i = 0; i < kMaxSpatialLayers; ++i) {
1059 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001060 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001061 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001062 for (int tid = 0; tid < num_layers; ++tid)
1063 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001064 }
1065 }
Erik Språngaed30702018-11-05 12:57:17 +01001066 }
Sergey Silkin6456e352019-07-08 17:56:40 +02001067
1068 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001069 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001070 info.apply_alignment_to_all_simulcast_layers =
1071 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001072 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001073 if (is_qp_trusted_.has_value()) {
1074 info.is_qp_trusted = is_qp_trusted_;
1075 }
Erik Språngaed30702018-11-05 12:57:17 +01001076 return info;
kthelgason876222f2016-11-29 01:44:11 -08001077 }
1078
Erik Språngb7cb7b52019-02-26 15:52:33 +01001079 int32_t RegisterEncodeCompleteCallback(
1080 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +02001081 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001082 encoded_image_callback_ = callback;
1083 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
1084 }
1085
perkjfa10b552016-10-02 23:45:26 -07001086 void ContinueEncode() { continue_encode_event_.Set(); }
1087
1088 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
1089 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +02001090 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -07001091 EXPECT_EQ(timestamp_, timestamp);
1092 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
1093 }
1094
kthelgason2fc52542017-03-03 00:24:41 -08001095 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +02001096 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -08001097 quality_scaling_ = b;
1098 }
kthelgasonad9010c2017-02-14 00:46:51 -08001099
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001100 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +02001101 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001102 requested_resolution_alignment_ = requested_resolution_alignment;
1103 }
1104
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001105 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
1106 MutexLock lock(&local_mutex_);
1107 apply_alignment_to_all_simulcast_layers_ = b;
1108 }
1109
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001110 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +02001111 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001112 is_hardware_accelerated_ = is_hardware_accelerated;
1113 }
1114
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001115 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
1116 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +02001117 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001118 temporal_layers_supported_[spatial_idx] = supported;
1119 }
1120
Sergey Silkin6456e352019-07-08 17:56:40 +02001121 void SetResolutionBitrateLimits(
1122 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +02001123 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +02001124 resolution_bitrate_limits_ = thresholds;
1125 }
1126
sprangfe627f32017-03-29 08:24:59 -07001127 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +02001128 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -07001129 force_init_encode_failed_ = force_failure;
1130 }
1131
Niels Möller6bb5ab92019-01-11 11:11:10 +01001132 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +02001133 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001134 rate_factor_ = rate_factor;
1135 }
1136
Erik Språngd7329ca2019-02-21 21:19:53 +01001137 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +02001138 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001139 return last_framerate_;
1140 }
1141
Erik Språngd7329ca2019-02-21 21:19:53 +01001142 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +02001143 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001144 return last_update_rect_;
1145 }
1146
Niels Möller87e2d782019-03-07 10:18:23 +01001147 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +02001148 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001149 return last_frame_types_;
1150 }
1151
1152 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +01001153 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +01001154 keyframe ? VideoFrameType::kVideoFrameKey
1155 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01001156 {
Markus Handella3765182020-07-08 13:13:32 +02001157 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001158 last_frame_types_ = frame_type;
1159 }
Niels Möllerb859b322019-03-07 12:40:01 +01001160 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +01001161 }
1162
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001163 void InjectEncodedImage(const EncodedImage& image,
1164 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +02001165 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001166 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001167 }
1168
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001169 void SetEncodedImageData(
1170 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +02001171 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001172 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001173 }
1174
Erik Språngd7329ca2019-02-21 21:19:53 +01001175 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +02001176 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001177 expect_null_frame_ = true;
1178 }
1179
Erik Språng5056af02019-09-02 15:53:11 +02001180 absl::optional<VideoEncoder::RateControlParameters>
1181 GetAndResetLastRateControlSettings() {
1182 auto settings = last_rate_control_settings_;
1183 last_rate_control_settings_.reset();
1184 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +01001185 }
1186
Henrik Boström56db9ff2021-03-24 09:06:45 +01001187 int GetLastInputWidth() const {
1188 MutexLock lock(&local_mutex_);
1189 return last_input_width_;
1190 }
1191
1192 int GetLastInputHeight() const {
1193 MutexLock lock(&local_mutex_);
1194 return last_input_height_;
1195 }
1196
Evan Shrubsole895556e2020-10-05 09:15:13 +02001197 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1198 MutexLock lock(&local_mutex_);
1199 return last_input_pixel_format_;
1200 }
1201
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001202 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001203 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001204 return num_set_rates_;
1205 }
1206
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001207 void SetPreferredPixelFormats(
1208 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1209 pixel_formats) {
1210 MutexLock lock(&local_mutex_);
1211 preferred_pixel_formats_ = std::move(pixel_formats);
1212 }
1213
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001214 void SetIsQpTrusted(absl::optional<bool> trusted) {
1215 MutexLock lock(&local_mutex_);
1216 is_qp_trusted_ = trusted;
1217 }
1218
Erik Språnge4589cb2022-04-06 16:44:30 +02001219 VideoCodecComplexity LastEncoderComplexity() {
1220 MutexLock lock(&local_mutex_);
1221 return last_encoder_complexity_;
1222 }
1223
perkjfa10b552016-10-02 23:45:26 -07001224 private:
perkj26091b12016-09-01 01:17:40 -07001225 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001226 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001227 {
Markus Handella3765182020-07-08 13:13:32 +02001228 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001229 if (expect_null_frame_) {
1230 EXPECT_EQ(input_image.timestamp(), 0u);
1231 EXPECT_EQ(input_image.width(), 1);
1232 last_frame_types_ = *frame_types;
1233 expect_null_frame_ = false;
1234 } else {
1235 EXPECT_GT(input_image.timestamp(), timestamp_);
1236 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1237 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1238 }
perkj26091b12016-09-01 01:17:40 -07001239
1240 timestamp_ = input_image.timestamp();
1241 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001242 last_input_width_ = input_image.width();
1243 last_input_height_ = input_image.height();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001244 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001245 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001246 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001247 }
Niels Möllerb859b322019-03-07 12:40:01 +01001248 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001249 return result;
1250 }
1251
Niels Möller08ae7ce2020-09-23 15:58:12 +02001252 CodecSpecificInfo EncodeHook(
1253 EncodedImage& encoded_image,
1254 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001255 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001256 {
1257 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001258 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001259 }
1260 MutexLock lock(&local_mutex_);
1261 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001262 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001263 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001264 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001265 }
1266
sprangfe627f32017-03-29 08:24:59 -07001267 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001268 const Settings& settings) override {
1269 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001270
Markus Handella3765182020-07-08 13:13:32 +02001271 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001272 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001273
Erik Språng82fad3d2018-03-21 09:57:23 +01001274 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001275 // Simulate setting up temporal layers, in order to validate the life
1276 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001277 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001278 frame_buffer_controller_ =
1279 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001280 }
Erik Språnge4589cb2022-04-06 16:44:30 +02001281
1282 last_encoder_complexity_ = config->GetVideoEncoderComplexity();
1283
Erik Språngb7cb7b52019-02-26 15:52:33 +01001284 if (force_init_encode_failed_) {
1285 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001286 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001287 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001288
Erik Språngb7cb7b52019-02-26 15:52:33 +01001289 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001290 return res;
1291 }
1292
Erik Språngb7cb7b52019-02-26 15:52:33 +01001293 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001294 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001295 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1296 initialized_ = EncoderState::kUninitialized;
1297 return FakeEncoder::Release();
1298 }
1299
Erik Språng16cb8f52019-04-12 13:59:09 +02001300 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001301 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001302 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001303 VideoBitrateAllocation adjusted_rate_allocation;
1304 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1305 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001306 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001307 adjusted_rate_allocation.SetBitrate(
1308 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001309 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001310 rate_factor_));
1311 }
1312 }
1313 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001314 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001315 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001316 RateControlParameters adjusted_paramters = parameters;
1317 adjusted_paramters.bitrate = adjusted_rate_allocation;
1318 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001319 }
1320
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001321 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001322 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001323 enum class EncoderState {
1324 kUninitialized,
1325 kInitializationFailed,
1326 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001327 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
perkj26091b12016-09-01 01:17:40 -07001328 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001329 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1330 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1331 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1332 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1333 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1334 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001335 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1336 false;
Markus Handella3765182020-07-08 13:13:32 +02001337 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001338 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1339 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001340 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001341 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001342 absl::optional<bool>
1343 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001344 local_mutex_);
1345 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1346 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1347 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001348 absl::optional<VideoEncoder::RateControlParameters>
1349 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001350 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1351 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001352 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001353 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001354 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1355 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001356 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 17:56:40 +02001357 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001358 RTC_GUARDED_BY(local_mutex_);
1359 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001360 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1361 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001362 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1363 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001364 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
Erik Språnge4589cb2022-04-06 16:44:30 +02001365 VideoCodecComplexity last_encoder_complexity_ RTC_GUARDED_BY(local_mutex_){
1366 VideoCodecComplexity::kComplexityNormal};
perkj26091b12016-09-01 01:17:40 -07001367 };
1368
mflodmancc3d4422017-08-03 08:27:51 -07001369 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001370 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001371 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1372 : time_controller_(time_controller), test_encoder_(test_encoder) {
1373 RTC_DCHECK(time_controller_);
1374 }
perkj26091b12016-09-01 01:17:40 -07001375
perkj26091b12016-09-01 01:17:40 -07001376 void WaitForEncodedFrame(int64_t expected_ntp_time) {
Markus Handell2cfc1af2022-08-19 08:16:48 +00001377 EXPECT_TRUE(TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeout));
sprang4847ae62017-06-27 07:06:52 -07001378 }
1379
1380 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
Markus Handell2cfc1af2022-08-19 08:16:48 +00001381 TimeDelta timeout) {
perkj26091b12016-09-01 01:17:40 -07001382 uint32_t timestamp = 0;
Markus Handell2cfc1af2022-08-19 08:16:48 +00001383 if (!WaitForFrame(timeout))
sprang4847ae62017-06-27 07:06:52 -07001384 return false;
perkj26091b12016-09-01 01:17:40 -07001385 {
Markus Handella3765182020-07-08 13:13:32 +02001386 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001387 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001388 }
1389 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001390 return true;
perkj26091b12016-09-01 01:17:40 -07001391 }
1392
sprangb1ca0732017-02-01 08:38:12 -08001393 void WaitForEncodedFrame(uint32_t expected_width,
1394 uint32_t expected_height) {
Markus Handell2cfc1af2022-08-19 08:16:48 +00001395 EXPECT_TRUE(WaitForFrame(kDefaultTimeout));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001396 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001397 }
1398
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001399 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001400 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001401 uint32_t width = 0;
1402 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001403 {
Markus Handella3765182020-07-08 13:13:32 +02001404 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001405 width = last_width_;
1406 height = last_height_;
1407 }
1408 EXPECT_EQ(expected_height, height);
1409 EXPECT_EQ(expected_width, width);
1410 }
1411
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001412 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1413 VideoRotation rotation;
1414 {
Markus Handella3765182020-07-08 13:13:32 +02001415 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001416 rotation = last_rotation_;
1417 }
1418 EXPECT_EQ(expected_rotation, rotation);
1419 }
1420
Markus Handell2cfc1af2022-08-19 08:16:48 +00001421 void ExpectDroppedFrame() {
1422 EXPECT_FALSE(WaitForFrame(TimeDelta::Millis(100)));
1423 }
kthelgason2bc68642017-02-07 07:02:22 -08001424
Markus Handell2cfc1af2022-08-19 08:16:48 +00001425 bool WaitForFrame(TimeDelta timeout) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001426 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +01001427 time_controller_->AdvanceTime(TimeDelta::Zero());
Markus Handell2cfc1af2022-08-19 08:16:48 +00001428 bool ret = encoded_frame_event_.Wait(timeout);
Markus Handell28c71802021-11-08 10:11:55 +01001429 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001430 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001431 }
1432
perkj26091b12016-09-01 01:17:40 -07001433 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001434 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001435 expect_frames_ = false;
1436 }
1437
asaperssonfab67072017-04-04 05:51:49 -07001438 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001439 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001440 return number_of_reconfigurations_;
1441 }
1442
asaperssonfab67072017-04-04 05:51:49 -07001443 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001444 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001445 return min_transmit_bitrate_bps_;
1446 }
1447
Erik Språngd7329ca2019-02-21 21:19:53 +01001448 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001449 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001450 num_expected_layers_ = num_layers;
1451 }
1452
Erik Språngb7cb7b52019-02-26 15:52:33 +01001453 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001454 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001455 return last_capture_time_ms_;
1456 }
1457
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001458 const EncodedImage& GetLastEncodedImage() {
1459 MutexLock lock(&mutex_);
1460 return last_encoded_image_;
1461 }
1462
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001463 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001464 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001465 return std::move(last_encoded_image_data_);
1466 }
1467
Per Kjellanderdcef6412020-10-07 15:09:05 +02001468 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1469 MutexLock lock(&mutex_);
1470 return last_bitrate_allocation_;
1471 }
1472
1473 int number_of_bitrate_allocations() const {
1474 MutexLock lock(&mutex_);
1475 return number_of_bitrate_allocations_;
1476 }
1477
Per Kjellandera9434842020-10-15 17:53:22 +02001478 VideoLayersAllocation GetLastVideoLayersAllocation() {
1479 MutexLock lock(&mutex_);
1480 return last_layers_allocation_;
1481 }
1482
1483 int number_of_layers_allocations() const {
1484 MutexLock lock(&mutex_);
1485 return number_of_layers_allocations_;
1486 }
1487
perkj26091b12016-09-01 01:17:40 -07001488 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001489 Result OnEncodedImage(
1490 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001491 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001492 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001493 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001494 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001495 last_encoded_image_data_ = std::vector<uint8_t>(
1496 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001497 uint32_t timestamp = encoded_image.Timestamp();
1498 if (last_timestamp_ != timestamp) {
1499 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001500 last_width_ = encoded_image._encodedWidth;
1501 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001502 } else {
1503 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001504 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1505 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001506 }
1507 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001508 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001509 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001510 if (num_received_layers_ == num_expected_layers_) {
1511 encoded_frame_event_.Set();
1512 }
sprangb1ca0732017-02-01 08:38:12 -08001513 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001514 }
1515
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001516 void OnEncoderConfigurationChanged(
1517 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001518 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001519 VideoEncoderConfig::ContentType content_type,
1520 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001521 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001522 ++number_of_reconfigurations_;
1523 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1524 }
1525
Per Kjellanderdcef6412020-10-07 15:09:05 +02001526 void OnBitrateAllocationUpdated(
1527 const VideoBitrateAllocation& allocation) override {
1528 MutexLock lock(&mutex_);
1529 ++number_of_bitrate_allocations_;
1530 last_bitrate_allocation_ = allocation;
1531 }
1532
Per Kjellandera9434842020-10-15 17:53:22 +02001533 void OnVideoLayersAllocationUpdated(
1534 VideoLayersAllocation allocation) override {
1535 MutexLock lock(&mutex_);
1536 ++number_of_layers_allocations_;
1537 last_layers_allocation_ = allocation;
1538 rtc::StringBuilder log;
1539 for (const auto& layer : allocation.active_spatial_layers) {
1540 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1541 << "[";
1542 for (const auto target_bitrate :
1543 layer.target_bitrate_per_temporal_layer) {
1544 log << target_bitrate.kbps() << ",";
1545 }
1546 log << "]";
1547 }
Harald Alvestrand97597c02021-11-04 12:01:23 +00001548 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 17:53:22 +02001549 }
1550
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001551 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001552 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001553 TestEncoder* test_encoder_;
1554 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001555 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001556 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001557 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001558 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001559 uint32_t last_height_ = 0;
1560 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001561 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001562 size_t num_expected_layers_ = 1;
1563 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001564 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001565 int number_of_reconfigurations_ = 0;
1566 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001567 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1568 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001569 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1570 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001571 };
1572
Sergey Silkin5ee69672019-07-02 14:18:34 +02001573 class VideoBitrateAllocatorProxyFactory
1574 : public VideoBitrateAllocatorFactory {
1575 public:
1576 VideoBitrateAllocatorProxyFactory()
1577 : bitrate_allocator_factory_(
1578 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1579
1580 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1581 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001582 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001583 codec_config_ = codec;
1584 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1585 }
1586
1587 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001588 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001589 return codec_config_;
1590 }
1591
1592 private:
1593 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1594
Markus Handella3765182020-07-08 13:13:32 +02001595 mutable Mutex mutex_;
1596 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001597 };
1598
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001599 Clock* clock() { return time_controller_.GetClock(); }
1600 void AdvanceTime(TimeDelta duration) {
1601 time_controller_.AdvanceTime(duration);
1602 }
1603
1604 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1605
1606 protected:
1607 virtual TaskQueueFactory* GetTaskQueueFactory() {
1608 return time_controller_.GetTaskQueueFactory();
1609 }
1610
Jonas Orelandc7f691a2022-03-09 15:12:07 +01001611 test::ScopedKeyValueConfig field_trials_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001612 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001613 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001614 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001615 int codec_width_;
1616 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001617 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001618 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001619 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001620 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001621 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001622 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001623 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001624 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001625};
1626
mflodmancc3d4422017-08-03 08:27:51 -07001627TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001628 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001629 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001630 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001631 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001632 WaitForEncodedFrame(1);
Markus Handell2cfc1af2022-08-19 08:16:48 +00001633 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
mflodmancc3d4422017-08-03 08:27:51 -07001634 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001635}
1636
mflodmancc3d4422017-08-03 08:27:51 -07001637TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001638 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001639 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001640 // The encoder will cache up to one frame for a short duration. Adding two
1641 // frames means that the first frame will be dropped and the second frame will
1642 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001643 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001644 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001645 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Markus Handell28c71802021-11-08 10:11:55 +01001646 AdvanceTime(TimeDelta::Zero());
Markus Handell2cfc1af2022-08-19 08:16:48 +00001647 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
perkj26091b12016-09-01 01:17:40 -07001648
Henrik Boström381d1092020-05-12 18:49:07 +02001649 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001650 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001651
Sebastian Janssona3177052018-04-10 13:05:49 +02001652 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001653 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001654 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1655
1656 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001657 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001658}
1659
mflodmancc3d4422017-08-03 08:27:51 -07001660TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001661 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001662 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001663 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001664 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001665
Henrik Boström381d1092020-05-12 18:49:07 +02001666 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001667 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1668
Sebastian Janssona3177052018-04-10 13:05:49 +02001669 // The encoder will cache up to one frame for a short duration. Adding two
1670 // frames means that the first frame will be dropped and the second frame will
1671 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001672 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001673 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001674
Henrik Boström381d1092020-05-12 18:49:07 +02001675 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001676 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001677 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001678 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1679 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001680 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001681}
1682
mflodmancc3d4422017-08-03 08:27:51 -07001683TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001684 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001685 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001686 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001687 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001688
1689 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001690 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001691
perkja49cbd32016-09-16 07:53:41 -07001692 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001693 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001694 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001695}
1696
mflodmancc3d4422017-08-03 08:27:51 -07001697TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001698 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001699 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001700
perkja49cbd32016-09-16 07:53:41 -07001701 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001702 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001703
mflodmancc3d4422017-08-03 08:27:51 -07001704 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001705 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001706 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001707 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
Markus Handell2cfc1af2022-08-19 08:16:48 +00001708 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
perkj26091b12016-09-01 01:17:40 -07001709}
1710
Markus Handell9a478b52021-11-18 16:07:01 +01001711TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
1712 test::FrameForwarder source;
1713 video_stream_encoder_->SetSource(&source,
1714 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02001715 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001716 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001717
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001718 int dropped_count = 0;
1719 stats_proxy_->SetDroppedFrameCallback(
1720 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1721 ++dropped_count;
1722 });
1723
Markus Handell9a478b52021-11-18 16:07:01 +01001724 source.IncomingCapturedFrame(CreateFrame(1, nullptr));
1725 source.IncomingCapturedFrame(CreateFrame(2, nullptr));
1726 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001727 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001728 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001729}
1730
Henrik Boström56db9ff2021-03-24 09:06:45 +01001731TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001732 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001733 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001734
1735 rtc::Event frame_destroyed_event;
1736 video_source_.IncomingCapturedFrame(
1737 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001738 WaitForEncodedFrame(1);
1739 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1740 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001741 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1742 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001743 video_stream_encoder_->Stop();
1744}
1745
Henrik Boström56db9ff2021-03-24 09:06:45 +01001746TEST_F(VideoStreamEncoderTest,
1747 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001748 // Use the cropping factory.
1749 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001750 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001751 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1752 kMaxPayloadLength);
1753 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1754
1755 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001756 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001757 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001758 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1759 WaitForEncodedFrame(1);
1760 // The encoder will have been configured once.
1761 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001762 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1763 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 06:59:12 -07001764
1765 // Now send in a fake frame that needs to be cropped as the width/height
1766 // aren't divisible by 4 (see CreateEncoderStreams above).
1767 rtc::Event frame_destroyed_event;
1768 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1769 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001770 WaitForEncodedFrame(2);
1771 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1772 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001773 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1774 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001775 video_stream_encoder_->Stop();
1776}
1777
Evan Shrubsole895556e2020-10-05 09:15:13 +02001778TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1779 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001780 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001781
1782 video_source_.IncomingCapturedFrame(
1783 CreateNV12Frame(1, codec_width_, codec_height_));
1784 WaitForEncodedFrame(1);
1785 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1786 fake_encoder_.GetLastInputPixelFormat());
1787 video_stream_encoder_->Stop();
1788}
1789
Henrik Boström56db9ff2021-03-24 09:06:45 +01001790TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001791 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001792 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001793
1794 fake_encoder_.SetPreferredPixelFormats({});
1795
1796 rtc::Event frame_destroyed_event;
1797 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1798 1, &frame_destroyed_event, codec_width_, codec_height_));
1799 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001800 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001801 fake_encoder_.GetLastInputPixelFormat());
1802 video_stream_encoder_->Stop();
1803}
1804
Henrik Boström56db9ff2021-03-24 09:06:45 +01001805TEST_F(VideoStreamEncoderTest,
1806 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001807 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001808 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001809
1810 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1811
1812 rtc::Event frame_destroyed_event;
1813 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1814 1, &frame_destroyed_event, codec_width_, codec_height_));
1815 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001816 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001817 fake_encoder_.GetLastInputPixelFormat());
1818 video_stream_encoder_->Stop();
1819}
1820
Henrik Boström56db9ff2021-03-24 09:06:45 +01001821TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001822 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001823 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001824
1825 // Fake NV12 native frame does not allow mapping to I444.
1826 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1827
1828 rtc::Event frame_destroyed_event;
1829 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1830 1, &frame_destroyed_event, codec_width_, codec_height_));
1831 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001832 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001833 fake_encoder_.GetLastInputPixelFormat());
1834 video_stream_encoder_->Stop();
1835}
1836
Henrik Boström56db9ff2021-03-24 09:06:45 +01001837TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001838 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001839 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001840
1841 rtc::Event frame_destroyed_event;
1842 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1843 1, &frame_destroyed_event, codec_width_, codec_height_));
1844 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001845 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001846 fake_encoder_.GetLastInputPixelFormat());
1847 video_stream_encoder_->Stop();
1848}
1849
Ying Wang9b881ab2020-02-07 14:29:32 +01001850TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001851 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001852 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001853 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1854 WaitForEncodedFrame(1);
1855
Henrik Boström381d1092020-05-12 18:49:07 +02001856 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001857 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001858 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1859 // frames. Adding two frames means that the first frame will be dropped and
1860 // the second frame will be sent to the encoder.
1861 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1862 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1863 WaitForEncodedFrame(3);
1864 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1865 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1866 WaitForEncodedFrame(5);
1867 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1868 video_stream_encoder_->Stop();
1869}
1870
mflodmancc3d4422017-08-03 08:27:51 -07001871TEST_F(VideoStreamEncoderTest,
1872 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001873 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001874 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001875 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001876
1877 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001878 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001879 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001880 // The encoder will have been configured once when the first frame is
1881 // received.
1882 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001883
1884 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001885 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001886 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001887 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001888 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001889
1890 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001891 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001892 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001893 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001894 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001895
mflodmancc3d4422017-08-03 08:27:51 -07001896 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001897}
1898
mflodmancc3d4422017-08-03 08:27:51 -07001899TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001900 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001901 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001902
1903 // Capture a frame and wait for it to synchronize with the encoder thread.
1904 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001905 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001906 // The encoder will have been configured once.
1907 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001908 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1909 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-02 23:45:26 -07001910
1911 codec_width_ *= 2;
1912 codec_height_ *= 2;
1913 // Capture a frame with a higher resolution and wait for it to synchronize
1914 // with the encoder thread.
1915 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001916 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 10:07:11 +02001917 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1918 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 21:37:57 +01001919 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001920
mflodmancc3d4422017-08-03 08:27:51 -07001921 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001922}
1923
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001924TEST_F(VideoStreamEncoderTest,
1925 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001926 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001927 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001928
1929 // Capture a frame and wait for it to synchronize with the encoder thread.
1930 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1931 WaitForEncodedFrame(1);
1932
1933 VideoEncoderConfig video_encoder_config;
1934 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1935 // Changing the max payload data length recreates encoder.
1936 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1937 kMaxPayloadLength / 2);
1938
1939 // Capture a frame and wait for it to synchronize with the encoder thread.
1940 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1941 WaitForEncodedFrame(2);
1942 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1943
1944 video_stream_encoder_->Stop();
1945}
1946
Sergey Silkin5ee69672019-07-02 14:18:34 +02001947TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001948 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001949 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001950
1951 VideoEncoderConfig video_encoder_config;
1952 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02001953 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1954 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001955 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1956 kMaxPayloadLength);
1957
1958 // Capture a frame and wait for it to synchronize with the encoder thread.
1959 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1960 WaitForEncodedFrame(1);
1961 // The encoder will have been configured once when the first frame is
1962 // received.
1963 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001964 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001965 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001966 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001967 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1968
Sergey Silkin6456e352019-07-08 17:56:40 +02001969 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1970 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 10:07:11 +02001971 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1972 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001973 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1974 kMaxPayloadLength);
1975
1976 // Capture a frame and wait for it to synchronize with the encoder thread.
1977 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1978 WaitForEncodedFrame(2);
1979 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1980 // Bitrate limits have changed - rate allocator should be reconfigured,
1981 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 10:07:11 +02001982 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001983 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001984 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001985 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001986 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001987
1988 video_stream_encoder_->Stop();
1989}
1990
Sergey Silkin6456e352019-07-08 17:56:40 +02001991TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001992 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001993 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001994 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001995
Sergey Silkincd02eba2020-01-20 14:48:40 +01001996 const uint32_t kMinEncBitrateKbps = 100;
1997 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001998 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001999 /*frame_size_pixels=*/codec_width_ * codec_height_,
2000 /*min_start_bitrate_bps=*/0,
2001 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
2002 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002003 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2004
Sergey Silkincd02eba2020-01-20 14:48:40 +01002005 VideoEncoderConfig video_encoder_config;
2006 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2007 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
2008 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2009 (kMinEncBitrateKbps + 1) * 1000;
2010 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2011 kMaxPayloadLength);
2012
2013 // When both encoder and app provide bitrate limits, the intersection of
2014 // provided sets should be used.
2015 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2016 WaitForEncodedFrame(1);
2017 EXPECT_EQ(kMaxEncBitrateKbps,
2018 bitrate_allocator_factory_.codec_config().maxBitrate);
2019 EXPECT_EQ(kMinEncBitrateKbps + 1,
2020 bitrate_allocator_factory_.codec_config().minBitrate);
2021
2022 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
2023 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2024 (kMinEncBitrateKbps - 1) * 1000;
2025 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2026 kMaxPayloadLength);
2027 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002028 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002029 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002030 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002031 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002032 bitrate_allocator_factory_.codec_config().minBitrate);
2033
Sergey Silkincd02eba2020-01-20 14:48:40 +01002034 video_stream_encoder_->Stop();
2035}
2036
2037TEST_F(VideoStreamEncoderTest,
2038 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02002039 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002040 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002041
2042 const uint32_t kMinAppBitrateKbps = 100;
2043 const uint32_t kMaxAppBitrateKbps = 200;
2044 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
2045 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
2046 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2047 /*frame_size_pixels=*/codec_width_ * codec_height_,
2048 /*min_start_bitrate_bps=*/0,
2049 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
2050 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
2051 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2052
2053 VideoEncoderConfig video_encoder_config;
2054 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2055 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
2056 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2057 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002058 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2059 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002060
Sergey Silkincd02eba2020-01-20 14:48:40 +01002061 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2062 WaitForEncodedFrame(1);
2063 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002064 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002065 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002066 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02002067
2068 video_stream_encoder_->Stop();
2069}
2070
2071TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002072 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02002073 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002074 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02002075
2076 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002077 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002078 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002079 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002080 fake_encoder_.SetResolutionBitrateLimits(
2081 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
2082
2083 VideoEncoderConfig video_encoder_config;
2084 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2085 video_encoder_config.max_bitrate_bps = 0;
2086 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2087 kMaxPayloadLength);
2088
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002089 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002090 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2091 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002092 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2093 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002094 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2095 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2096
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002097 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002098 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2099 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002100 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2101 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002102 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2103 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2104
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002105 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02002106 // encoder for 360p should be used.
2107 video_source_.IncomingCapturedFrame(
2108 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2109 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002110 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2111 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002112 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2113 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2114
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002115 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02002116 // ignored.
2117 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2118 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002119 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2120 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002121 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2122 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002123 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2124 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002125 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2126 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2127
2128 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2129 // for 270p should be used.
2130 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2131 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002132 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2133 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002134 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2135 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2136
2137 video_stream_encoder_->Stop();
2138}
2139
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002140TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002141 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002142 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002143
2144 VideoEncoderConfig video_encoder_config;
2145 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2146 video_encoder_config.max_bitrate_bps = 0;
2147 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2148 kMaxPayloadLength);
2149
2150 // Encode 720p frame to get the default encoder target bitrate.
2151 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2152 WaitForEncodedFrame(1);
2153 const uint32_t kDefaultTargetBitrateFor720pKbps =
2154 bitrate_allocator_factory_.codec_config()
2155 .simulcastStream[0]
2156 .targetBitrate;
2157
2158 // Set the max recommended encoder bitrate to something lower than the default
2159 // target bitrate.
2160 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2161 1280 * 720, 10 * 1000, 10 * 1000,
2162 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2163 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2164
2165 // Change resolution to trigger encoder reinitialization.
2166 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2167 WaitForEncodedFrame(2);
2168 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2169 WaitForEncodedFrame(3);
2170
2171 // Ensure the target bitrate is capped by the max bitrate.
2172 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2173 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2174 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2175 .simulcastStream[0]
2176 .targetBitrate *
2177 1000,
2178 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2179
2180 video_stream_encoder_->Stop();
2181}
2182
Åsa Perssona7e34d32021-01-20 15:36:13 +01002183TEST_F(VideoStreamEncoderTest,
2184 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2185 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2186 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2187 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2188 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2189 fake_encoder_.SetResolutionBitrateLimits(
2190 {kEncoderLimits270p, kEncoderLimits360p});
2191
2192 // Two streams, highest stream active.
2193 VideoEncoderConfig config;
Jonas Oreland80c87d72022-09-29 15:01:09 +02002194 webrtc::VideoEncoder::EncoderInfo encoder_info;
Åsa Perssona7e34d32021-01-20 15:36:13 +01002195 const int kNumStreams = 2;
2196 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2197 config.max_bitrate_bps = 0;
2198 config.simulcast_layers[0].active = false;
2199 config.simulcast_layers[1].active = true;
2200 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002201 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002202 "VP8", /*max qp*/ 56, /*screencast*/ false,
Jonas Oreland80c87d72022-09-29 15:01:09 +02002203 /*screenshare enabled*/ false, encoder_info);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002204 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2205
2206 // The encoder bitrate limits for 270p should be used.
2207 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
Erik Språngf449af82022-08-08 17:54:55 +02002208 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002209 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002210 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002211 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002212 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002213 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002214
2215 // The encoder bitrate limits for 360p should be used.
2216 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
Erik Språngf449af82022-08-08 17:54:55 +02002217 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002218 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002219 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002220 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002221 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002222
2223 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2224 video_source_.IncomingCapturedFrame(
2225 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
Erik Språngf449af82022-08-08 17:54:55 +02002226 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002227 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002228 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002229 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002230 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002231
2232 // Resolution higher than 360p. Encoder limits should be ignored.
2233 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02002234 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002235 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002236 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002237 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002238 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002239 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002240 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002241 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002242 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002243
2244 // Resolution lower than 270p. The encoder limits for 270p should be used.
2245 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
Erik Språngf449af82022-08-08 17:54:55 +02002246 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002247 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002248 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002249 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002250 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002251
2252 video_stream_encoder_->Stop();
2253}
2254
2255TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002256 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2257 // Two streams, highest stream active.
2258 VideoEncoderConfig config;
Jonas Oreland80c87d72022-09-29 15:01:09 +02002259 webrtc::VideoEncoder::EncoderInfo encoder_info;
Åsa Persson258e9892021-02-25 10:39:51 +01002260 const int kNumStreams = 2;
2261 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2262 config.max_bitrate_bps = 0;
2263 config.simulcast_layers[0].active = false;
2264 config.simulcast_layers[1].active = true;
2265 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002266 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002267 "VP8", /*max qp*/ 56, /*screencast*/ false,
Jonas Oreland80c87d72022-09-29 15:01:09 +02002268 /*screenshare enabled*/ false, encoder_info);
Åsa Persson258e9892021-02-25 10:39:51 +01002269 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2270
2271 // Default bitrate limits for 270p should be used.
2272 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2273 kDefaultLimits270p =
2274 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002275 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002276 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
Erik Språngf449af82022-08-08 17:54:55 +02002277 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002278 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 10:39:51 +01002279 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002280 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002281 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002282 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002283
2284 // Default bitrate limits for 360p should be used.
2285 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2286 kDefaultLimits360p =
2287 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002288 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002289 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
Erik Språngf449af82022-08-08 17:54:55 +02002290 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Persson258e9892021-02-25 10:39:51 +01002291 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002292 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002293 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002294 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002295
2296 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2297 video_source_.IncomingCapturedFrame(
2298 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
Erik Språngf449af82022-08-08 17:54:55 +02002299 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Persson258e9892021-02-25 10:39:51 +01002300 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002301 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002302 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002303 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002304
2305 // Default bitrate limits for 540p should be used.
2306 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2307 kDefaultLimits540p =
2308 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002309 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002310 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02002311 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Persson258e9892021-02-25 10:39:51 +01002312 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002313 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002314 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002315 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002316
2317 video_stream_encoder_->Stop();
2318}
2319
2320TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002321 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2322 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2323 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2324 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2325 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2326 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2327 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2328 fake_encoder_.SetResolutionBitrateLimits(
2329 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2330
2331 // Three streams, middle stream active.
2332 VideoEncoderConfig config;
Jonas Oreland80c87d72022-09-29 15:01:09 +02002333 webrtc::VideoEncoder::EncoderInfo encoder_info;
Åsa Perssona7e34d32021-01-20 15:36:13 +01002334 const int kNumStreams = 3;
2335 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2336 config.simulcast_layers[0].active = false;
2337 config.simulcast_layers[1].active = true;
2338 config.simulcast_layers[2].active = false;
2339 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002340 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002341 "VP8", /*max qp*/ 56, /*screencast*/ false,
Jonas Oreland80c87d72022-09-29 15:01:09 +02002342 /*screenshare enabled*/ false, encoder_info);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002343 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2344
2345 // The encoder bitrate limits for 360p should be used.
2346 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02002347 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002348 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002349 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002350 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002351 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002352 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002353
2354 // The encoder bitrate limits for 270p should be used.
2355 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02002356 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002357 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002358 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002359 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002360 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002361
2362 video_stream_encoder_->Stop();
2363}
2364
2365TEST_F(VideoStreamEncoderTest,
2366 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2367 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2368 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2369 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2370 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2371 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2372 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2373 fake_encoder_.SetResolutionBitrateLimits(
2374 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2375
2376 // Three streams, lowest stream active.
2377 VideoEncoderConfig config;
Jonas Oreland80c87d72022-09-29 15:01:09 +02002378 webrtc::VideoEncoder::EncoderInfo encoder_info;
Åsa Perssona7e34d32021-01-20 15:36:13 +01002379 const int kNumStreams = 3;
2380 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2381 config.simulcast_layers[0].active = true;
2382 config.simulcast_layers[1].active = false;
2383 config.simulcast_layers[2].active = false;
2384 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002385 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002386 "VP8", /*max qp*/ 56, /*screencast*/ false,
Jonas Oreland80c87d72022-09-29 15:01:09 +02002387 /*screenshare enabled*/ false, encoder_info);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002388 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2389
2390 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2391 // on lowest stream, limits for 270p should not be used
2392 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02002393 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002394 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002395 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002396 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002397 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002398 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002399
2400 video_stream_encoder_->Stop();
2401}
2402
2403TEST_F(VideoStreamEncoderTest,
2404 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2405 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2406 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2407 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2408 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2409 fake_encoder_.SetResolutionBitrateLimits(
2410 {kEncoderLimits270p, kEncoderLimits360p});
2411 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2412
2413 // Two streams, highest stream active.
2414 VideoEncoderConfig config;
Jonas Oreland80c87d72022-09-29 15:01:09 +02002415 webrtc::VideoEncoder::EncoderInfo encoder_info;
Åsa Perssona7e34d32021-01-20 15:36:13 +01002416 const int kNumStreams = 2;
2417 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2418 config.simulcast_layers[0].active = false;
2419 config.simulcast_layers[1].active = true;
2420 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2421 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002422 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002423 "VP8", /*max qp*/ 56, /*screencast*/ false,
Jonas Oreland80c87d72022-09-29 15:01:09 +02002424 /*screenshare enabled*/ false, encoder_info);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002425 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2426
2427 // The encoder bitrate limits for 270p should be used.
2428 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
Erik Språngf449af82022-08-08 17:54:55 +02002429 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02002430 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002431 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002432 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002433 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002434 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002435
2436 // The max configured bitrate is less than the encoder limit for 360p.
2437 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
Erik Språngf449af82022-08-08 17:54:55 +02002438 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Åsa Perssona7e34d32021-01-20 15:36:13 +01002439 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002440 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002441 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002442 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002443
2444 video_stream_encoder_->Stop();
2445}
2446
mflodmancc3d4422017-08-03 08:27:51 -07002447TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002448 EXPECT_TRUE(video_source_.has_sinks());
2449 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002450 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002451 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002452 EXPECT_FALSE(video_source_.has_sinks());
2453 EXPECT_TRUE(new_video_source.has_sinks());
2454
mflodmancc3d4422017-08-03 08:27:51 -07002455 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002456}
2457
mflodmancc3d4422017-08-03 08:27:51 -07002458TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002459 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002460 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002461 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002462 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002463}
2464
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002465class ResolutionAlignmentTest
2466 : public VideoStreamEncoderTest,
2467 public ::testing::WithParamInterface<
2468 ::testing::tuple<int, std::vector<double>>> {
2469 public:
2470 ResolutionAlignmentTest()
2471 : requested_alignment_(::testing::get<0>(GetParam())),
2472 scale_factors_(::testing::get<1>(GetParam())) {}
2473
2474 protected:
2475 const int requested_alignment_;
2476 const std::vector<double> scale_factors_;
2477};
2478
2479INSTANTIATE_TEST_SUITE_P(
2480 AlignmentAndScaleFactors,
2481 ResolutionAlignmentTest,
2482 ::testing::Combine(
2483 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2484 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2485 std::vector<double>{-1.0, -1.0},
2486 std::vector<double>{-1.0, -1.0, -1.0},
2487 std::vector<double>{4.0, 2.0, 1.0},
2488 std::vector<double>{9999.0, -1.0, 1.0},
2489 std::vector<double>{3.99, 2.01, 1.0},
2490 std::vector<double>{4.9, 1.7, 1.25},
2491 std::vector<double>{10.0, 4.0, 3.0},
2492 std::vector<double>{1.75, 3.5},
2493 std::vector<double>{1.5, 2.5},
2494 std::vector<double>{1.3, 1.0})));
2495
2496TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2497 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002498 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002499 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2500 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2501
2502 // Fill config with the scaling factor by which to reduce encoding size.
2503 const int num_streams = scale_factors_.size();
2504 VideoEncoderConfig config;
Jonas Oreland80c87d72022-09-29 15:01:09 +02002505 webrtc::VideoEncoder::EncoderInfo encoder_info;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002506 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2507 for (int i = 0; i < num_streams; ++i) {
2508 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2509 }
2510 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002511 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002512 "VP8", /*max qp*/ 56, /*screencast*/ false,
Jonas Oreland80c87d72022-09-29 15:01:09 +02002513 /*screenshare enabled*/ false, encoder_info);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002514 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2515
Henrik Boström381d1092020-05-12 18:49:07 +02002516 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002517 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2518 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002519 // Wait for all layers before triggering event.
2520 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002521
2522 // On the 1st frame, we should have initialized the encoder and
2523 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002524 int64_t timestamp_ms = kFrameIntervalMs;
2525 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2526 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002527 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002528
2529 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2530 // (It's up the to the encoder to potentially drop the previous frame,
2531 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002532 timestamp_ms += kFrameIntervalMs;
2533 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2534 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002535 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002536
Asa Persson606d3cb2021-10-04 10:07:11 +02002537 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002538 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2539 // Frame size should be a multiple of the requested alignment.
2540 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2541 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2542 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2543 // Aspect ratio should match.
2544 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2545 codec.height * codec.simulcastStream[i].width);
2546 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002547
2548 video_stream_encoder_->Stop();
2549}
2550
Jonathan Yubc771b72017-12-08 17:04:29 -08002551TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2552 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002553 const int kWidth = 1280;
2554 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002555
2556 // We rely on the automatic resolution adaptation, but we handle framerate
2557 // adaptation manually by mocking the stats proxy.
2558 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002559
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002560 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002561 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002562 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002563 video_stream_encoder_->SetSource(&video_source_,
2564 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002565 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002566 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002567 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002568 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2569
Jonathan Yubc771b72017-12-08 17:04:29 -08002570 // Adapt down as far as possible.
2571 rtc::VideoSinkWants last_wants;
2572 int64_t t = 1;
2573 int loop_count = 0;
2574 do {
2575 ++loop_count;
2576 last_wants = video_source_.sink_wants();
2577
2578 // Simulate the framerate we've been asked to adapt to.
2579 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2580 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2581 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2582 mock_stats.input_frame_rate = fps;
2583 stats_proxy_->SetMockStats(mock_stats);
2584
2585 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2586 sink_.WaitForEncodedFrame(t);
2587 t += frame_interval_ms;
2588
mflodmancc3d4422017-08-03 08:27:51 -07002589 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002590 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002591 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002592 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2593 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002594 } while (video_source_.sink_wants().max_pixel_count <
2595 last_wants.max_pixel_count ||
2596 video_source_.sink_wants().max_framerate_fps <
2597 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002598
Jonathan Yubc771b72017-12-08 17:04:29 -08002599 // Verify that we've adapted all the way down.
2600 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002601 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002602 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2603 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002604 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002605 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2606 *video_source_.last_sent_height());
2607 EXPECT_EQ(kMinBalancedFramerateFps,
2608 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002609
Jonathan Yubc771b72017-12-08 17:04:29 -08002610 // Adapt back up the same number of times we adapted down.
2611 for (int i = 0; i < loop_count - 1; ++i) {
2612 last_wants = video_source_.sink_wants();
2613
2614 // Simulate the framerate we've been asked to adapt to.
2615 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2616 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2617 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2618 mock_stats.input_frame_rate = fps;
2619 stats_proxy_->SetMockStats(mock_stats);
2620
2621 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2622 sink_.WaitForEncodedFrame(t);
2623 t += frame_interval_ms;
2624
Henrik Boström91aa7322020-04-28 12:24:33 +02002625 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002626 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002627 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002628 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2629 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002630 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2631 last_wants.max_pixel_count ||
2632 video_source_.sink_wants().max_framerate_fps >
2633 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002634 }
2635
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002636 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002637 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002638 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002639 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2640 EXPECT_EQ((loop_count - 1) * 2,
2641 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002642
mflodmancc3d4422017-08-03 08:27:51 -07002643 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002644}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002645
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002646TEST_F(VideoStreamEncoderTest,
2647 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02002648 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2649 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002650 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002651
2652 const int kFrameWidth = 1280;
2653 const int kFrameHeight = 720;
2654
2655 int64_t ntp_time = kFrameIntervalMs;
2656
2657 // Force an input frame rate to be available, or the adaptation call won't
2658 // know what framerate to adapt form.
2659 const int kInputFps = 30;
2660 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2661 stats.input_frame_rate = kInputFps;
2662 stats_proxy_->SetMockStats(stats);
2663
2664 video_source_.set_adaptation_enabled(true);
2665 video_stream_encoder_->SetSource(
2666 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002667 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002668 video_source_.IncomingCapturedFrame(
2669 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2670 sink_.WaitForEncodedFrame(ntp_time);
2671 ntp_time += kFrameIntervalMs;
2672
2673 // Trigger CPU overuse.
2674 video_stream_encoder_->TriggerCpuOveruse();
2675 video_source_.IncomingCapturedFrame(
2676 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2677 sink_.WaitForEncodedFrame(ntp_time);
2678 ntp_time += kFrameIntervalMs;
2679
2680 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2681 EXPECT_EQ(std::numeric_limits<int>::max(),
2682 video_source_.sink_wants().max_pixel_count);
2683 // Some framerate constraint should be set.
2684 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2685 EXPECT_LT(restricted_fps, kInputFps);
2686 video_source_.IncomingCapturedFrame(
2687 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2688 sink_.WaitForEncodedFrame(ntp_time);
2689 ntp_time += 100;
2690
Henrik Boström2671dac2020-05-19 16:29:09 +02002691 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002692 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2693 // Give the encoder queue time to process the change in degradation preference
2694 // by waiting for an encoded frame.
2695 video_source_.IncomingCapturedFrame(
2696 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2697 sink_.WaitForEncodedFrame(ntp_time);
2698 ntp_time += kFrameIntervalMs;
2699
2700 video_stream_encoder_->TriggerQualityLow();
2701 video_source_.IncomingCapturedFrame(
2702 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2703 sink_.WaitForEncodedFrame(ntp_time);
2704 ntp_time += kFrameIntervalMs;
2705
2706 // Some resolution constraint should be set.
2707 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2708 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2709 kFrameWidth * kFrameHeight);
2710 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2711
2712 int pixel_count = video_source_.sink_wants().max_pixel_count;
2713 // Triggering a CPU underuse should not change the sink wants since it has
2714 // not been overused for resolution since we changed degradation preference.
2715 video_stream_encoder_->TriggerCpuUnderuse();
2716 video_source_.IncomingCapturedFrame(
2717 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2718 sink_.WaitForEncodedFrame(ntp_time);
2719 ntp_time += kFrameIntervalMs;
2720 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2721 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2722
Evan Shrubsole64469032020-06-11 10:45:29 +02002723 // Change the degradation preference back. CPU underuse should not adapt since
2724 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002725 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002726 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2727 video_source_.IncomingCapturedFrame(
2728 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2729 sink_.WaitForEncodedFrame(ntp_time);
2730 ntp_time += 100;
2731 // Resolution adaptations is gone after changing degradation preference.
2732 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2733 EXPECT_EQ(std::numeric_limits<int>::max(),
2734 video_source_.sink_wants().max_pixel_count);
2735 // The fps adaptation from above is now back.
2736 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2737
2738 // Trigger CPU underuse.
2739 video_stream_encoder_->TriggerCpuUnderuse();
2740 video_source_.IncomingCapturedFrame(
2741 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2742 sink_.WaitForEncodedFrame(ntp_time);
2743 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002744 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2745
2746 // Trigger QP underuse, fps should return to normal.
2747 video_stream_encoder_->TriggerQualityHigh();
2748 video_source_.IncomingCapturedFrame(
2749 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2750 sink_.WaitForEncodedFrame(ntp_time);
2751 ntp_time += kFrameIntervalMs;
2752 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002753
2754 video_stream_encoder_->Stop();
2755}
2756
mflodmancc3d4422017-08-03 08:27:51 -07002757TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002758 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002759 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002760 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002761
sprangc5d62e22017-04-02 23:53:04 -07002762 const int kFrameWidth = 1280;
2763 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002764
Åsa Persson8c1bf952018-09-13 10:42:19 +02002765 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002766
kthelgason5e13d412016-12-01 03:59:51 -08002767 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002768 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002769 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002770 frame_timestamp += kFrameIntervalMs;
2771
perkj803d97f2016-11-01 11:45:46 -07002772 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002773 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002774 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002775 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002776 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002777 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002778
asapersson0944a802017-04-07 00:57:58 -07002779 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002780 // wanted resolution.
2781 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2782 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2783 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002784 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002785
2786 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002787 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002788 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002789 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002790 // Give the encoder queue time to process the change in degradation preference
2791 // by waiting for an encoded frame.
2792 new_video_source.IncomingCapturedFrame(
2793 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2794 sink_.WaitForEncodedFrame(frame_timestamp);
2795 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002796 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002797 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002798
sprangc5d62e22017-04-02 23:53:04 -07002799 // Force an input frame rate to be available, or the adaptation call won't
2800 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002801 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002802 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002803 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002804 stats_proxy_->SetMockStats(stats);
2805
mflodmancc3d4422017-08-03 08:27:51 -07002806 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002807 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002808 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002809 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002810 frame_timestamp += kFrameIntervalMs;
2811
2812 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002813 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002814 EXPECT_EQ(std::numeric_limits<int>::max(),
2815 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002816 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002817
asapersson02465b82017-04-10 01:12:52 -07002818 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002819 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2820 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002821 // Give the encoder queue time to process the change in degradation preference
2822 // by waiting for an encoded frame.
2823 new_video_source.IncomingCapturedFrame(
2824 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2825 sink_.WaitForEncodedFrame(frame_timestamp);
2826 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002827 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002828
mflodmancc3d4422017-08-03 08:27:51 -07002829 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002830 new_video_source.IncomingCapturedFrame(
2831 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002832 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002833 frame_timestamp += kFrameIntervalMs;
2834
2835 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002836 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002837
2838 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002839 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002840 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002841 // Give the encoder queue time to process the change in degradation preference
2842 // by waiting for an encoded frame.
2843 new_video_source.IncomingCapturedFrame(
2844 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2845 sink_.WaitForEncodedFrame(frame_timestamp);
2846 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002847 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2848 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002849 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002850 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002851
2852 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002853 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002854 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002855 // Give the encoder queue time to process the change in degradation preference
2856 // by waiting for an encoded frame.
2857 new_video_source.IncomingCapturedFrame(
2858 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2859 sink_.WaitForEncodedFrame(frame_timestamp);
2860 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002861 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2862 EXPECT_EQ(std::numeric_limits<int>::max(),
2863 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002864 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002865
mflodmancc3d4422017-08-03 08:27:51 -07002866 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002867}
2868
mflodmancc3d4422017-08-03 08:27:51 -07002869TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002870 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002871 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002872
asaperssonfab67072017-04-04 05:51:49 -07002873 const int kWidth = 1280;
2874 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002875 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002876 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002877 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2878 EXPECT_FALSE(stats.bw_limited_resolution);
2879 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2880
2881 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002882 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002883 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002884 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002885
2886 stats = stats_proxy_->GetStats();
2887 EXPECT_TRUE(stats.bw_limited_resolution);
2888 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2889
2890 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002891 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002892 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002893 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002894
2895 stats = stats_proxy_->GetStats();
2896 EXPECT_FALSE(stats.bw_limited_resolution);
2897 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2898 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2899
mflodmancc3d4422017-08-03 08:27:51 -07002900 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002901}
2902
mflodmancc3d4422017-08-03 08:27:51 -07002903TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002904 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002905 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002906
2907 const int kWidth = 1280;
2908 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002909 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002910 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002911 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2912 EXPECT_FALSE(stats.cpu_limited_resolution);
2913 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2914
2915 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002916 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002917 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002918 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002919
2920 stats = stats_proxy_->GetStats();
2921 EXPECT_TRUE(stats.cpu_limited_resolution);
2922 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2923
2924 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002925 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002926 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002927 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002928
2929 stats = stats_proxy_->GetStats();
2930 EXPECT_FALSE(stats.cpu_limited_resolution);
2931 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002932 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002933
mflodmancc3d4422017-08-03 08:27:51 -07002934 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002935}
2936
mflodmancc3d4422017-08-03 08:27:51 -07002937TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002938 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002939 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002940
asaperssonfab67072017-04-04 05:51:49 -07002941 const int kWidth = 1280;
2942 const int kHeight = 720;
2943 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002944 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002945 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002946 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002947 EXPECT_FALSE(stats.cpu_limited_resolution);
2948 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2949
asaperssonfab67072017-04-04 05:51:49 -07002950 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002951 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002952 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002953 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002954 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002955 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002956 EXPECT_TRUE(stats.cpu_limited_resolution);
2957 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2958
2959 // Set new source with adaptation still enabled.
2960 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002961 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002962 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002963
asaperssonfab67072017-04-04 05:51:49 -07002964 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002965 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002966 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002967 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002968 EXPECT_TRUE(stats.cpu_limited_resolution);
2969 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2970
2971 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002972 video_stream_encoder_->SetSource(&new_video_source,
2973 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002974
asaperssonfab67072017-04-04 05:51:49 -07002975 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002976 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002977 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002978 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002979 EXPECT_FALSE(stats.cpu_limited_resolution);
2980 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2981
2982 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002983 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002984 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002985
asaperssonfab67072017-04-04 05:51:49 -07002986 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002987 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002988 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002989 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002990 EXPECT_TRUE(stats.cpu_limited_resolution);
2991 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2992
asaperssonfab67072017-04-04 05:51:49 -07002993 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002994 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002995 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002996 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002997 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002998 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002999 EXPECT_FALSE(stats.cpu_limited_resolution);
3000 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003001 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003002
mflodmancc3d4422017-08-03 08:27:51 -07003003 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003004}
3005
mflodmancc3d4422017-08-03 08:27:51 -07003006TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02003007 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003008 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003009
asaperssonfab67072017-04-04 05:51:49 -07003010 const int kWidth = 1280;
3011 const int kHeight = 720;
3012 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003013 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003014 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003015 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003016 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003017 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003018
3019 // Set new source with adaptation still enabled.
3020 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003021 video_stream_encoder_->SetSource(&new_video_source,
3022 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08003023
asaperssonfab67072017-04-04 05:51:49 -07003024 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003025 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08003026 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003027 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003028 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003029 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003030
asaperssonfab67072017-04-04 05:51:49 -07003031 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003032 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003033 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003034 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003035 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003036 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003037 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003038 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003039
asaperssonfab67072017-04-04 05:51:49 -07003040 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003041 video_stream_encoder_->SetSource(&new_video_source,
3042 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08003043
asaperssonfab67072017-04-04 05:51:49 -07003044 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003045 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003046 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003047 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003048 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003049 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003050
asapersson02465b82017-04-10 01:12:52 -07003051 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07003052 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003053 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003054
asaperssonfab67072017-04-04 05:51:49 -07003055 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003056 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08003057 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003058 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003059 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003060 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
3061 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003062
mflodmancc3d4422017-08-03 08:27:51 -07003063 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003064}
3065
mflodmancc3d4422017-08-03 08:27:51 -07003066TEST_F(VideoStreamEncoderTest,
3067 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02003068 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003069 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07003070
3071 const int kWidth = 1280;
3072 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02003073 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07003074 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003075 video_source_.IncomingCapturedFrame(
3076 CreateFrame(timestamp_ms, kWidth, kHeight));
3077 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003078 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3079 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3080 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3081
3082 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003083 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003084 timestamp_ms += kFrameIntervalMs;
3085 video_source_.IncomingCapturedFrame(
3086 CreateFrame(timestamp_ms, kWidth, kHeight));
3087 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003088 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3089 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3090 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3091
3092 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07003093 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003094 timestamp_ms += kFrameIntervalMs;
3095 video_source_.IncomingCapturedFrame(
3096 CreateFrame(timestamp_ms, kWidth, kHeight));
3097 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003098 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3099 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3100 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3101
Niels Möller4db138e2018-04-19 09:04:13 +02003102 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07003103 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003104
3105 VideoEncoderConfig video_encoder_config;
3106 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3107 // Make format different, to force recreation of encoder.
3108 video_encoder_config.video_format.parameters["foo"] = "foo";
3109 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003110 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003111 timestamp_ms += kFrameIntervalMs;
3112 video_source_.IncomingCapturedFrame(
3113 CreateFrame(timestamp_ms, kWidth, kHeight));
3114 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003115 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3116 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3117 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3118
mflodmancc3d4422017-08-03 08:27:51 -07003119 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003120}
3121
mflodmancc3d4422017-08-03 08:27:51 -07003122TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003123 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003124 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003125 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003126
3127 const int kWidth = 1280;
3128 const int kHeight = 720;
3129 int sequence = 1;
3130
3131 // Enable BALANCED preference, no initial limitation.
3132 test::FrameForwarder source;
3133 video_stream_encoder_->SetSource(&source,
3134 webrtc::DegradationPreference::BALANCED);
3135 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3136 WaitForEncodedFrame(sequence++);
3137 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3138 EXPECT_FALSE(stats.cpu_limited_resolution);
3139 EXPECT_FALSE(stats.cpu_limited_framerate);
3140 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3141
3142 // Trigger CPU overuse, should now adapt down.
3143 video_stream_encoder_->TriggerCpuOveruse();
3144 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3145 WaitForEncodedFrame(sequence++);
3146 stats = stats_proxy_->GetStats();
3147 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3148
3149 // Set new degradation preference should clear restrictions since we changed
3150 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003151 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003152 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3153 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3154 WaitForEncodedFrame(sequence++);
3155 stats = stats_proxy_->GetStats();
3156 EXPECT_FALSE(stats.cpu_limited_resolution);
3157 EXPECT_FALSE(stats.cpu_limited_framerate);
3158 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3159
3160 // Force an input frame rate to be available, or the adaptation call won't
3161 // know what framerate to adapt from.
3162 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3163 mock_stats.input_frame_rate = 30;
3164 stats_proxy_->SetMockStats(mock_stats);
3165 video_stream_encoder_->TriggerCpuOveruse();
3166 stats_proxy_->ResetMockStats();
3167 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3168 WaitForEncodedFrame(sequence++);
3169
3170 // We have now adapted once.
3171 stats = stats_proxy_->GetStats();
3172 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3173
3174 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003175 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3176 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003177 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3178 WaitForEncodedFrame(sequence++);
3179 stats = stats_proxy_->GetStats();
3180 EXPECT_FALSE(stats.cpu_limited_resolution);
3181 EXPECT_FALSE(stats.cpu_limited_framerate);
3182 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3183
3184 video_stream_encoder_->Stop();
3185}
3186
3187TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003188 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003189 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003190 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003191
asapersson0944a802017-04-07 00:57:58 -07003192 const int kWidth = 1280;
3193 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003194 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003195
asaperssonfab67072017-04-04 05:51:49 -07003196 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003197 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003198 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003199 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003200 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003201 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3202
asapersson02465b82017-04-10 01:12:52 -07003203 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003204 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003205 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003206 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003207 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003208 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003209 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003210 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3211
3212 // Set new source with adaptation still enabled.
3213 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003214 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003215 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003216
3217 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003218 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003219 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003220 stats = stats_proxy_->GetStats();
3221 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003222 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003223 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3224
sprangc5d62e22017-04-02 23:53:04 -07003225 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003226 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003227 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003228 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003229 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003230 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003231 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003232 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003233 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003234 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003235 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3236
sprangc5d62e22017-04-02 23:53:04 -07003237 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003238 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003239 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3240 mock_stats.input_frame_rate = 30;
3241 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003242 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003243 stats_proxy_->ResetMockStats();
3244
3245 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003246 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003247 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003248
3249 // Framerate now adapted.
3250 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003251 EXPECT_FALSE(stats.cpu_limited_resolution);
3252 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003253 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3254
3255 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003256 video_stream_encoder_->SetSource(&new_video_source,
3257 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003258 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003259 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003260 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003261
3262 stats = stats_proxy_->GetStats();
3263 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003264 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003265 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3266
3267 // Try to trigger overuse. Should not succeed.
3268 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003269 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003270 stats_proxy_->ResetMockStats();
3271
3272 stats = stats_proxy_->GetStats();
3273 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003274 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003275 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3276
3277 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003278 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003279 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003280 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003281 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003282 stats = stats_proxy_->GetStats();
3283 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003284 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003285 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003286
3287 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003288 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003289 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003290 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003291 stats = stats_proxy_->GetStats();
3292 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003293 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003294 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3295
3296 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003297 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003298 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003299 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003300 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003301 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003302 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003303 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003304 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003305 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003306 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3307
3308 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003309 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003310 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003311 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003312 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003313 stats = stats_proxy_->GetStats();
3314 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003315 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003316 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003317 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003318
mflodmancc3d4422017-08-03 08:27:51 -07003319 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003320}
3321
mflodmancc3d4422017-08-03 08:27:51 -07003322TEST_F(VideoStreamEncoderTest,
3323 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003324 const int kWidth = 1280;
3325 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003326 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003327 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003328
asaperssonfab67072017-04-04 05:51:49 -07003329 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003330 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003331
asaperssonfab67072017-04-04 05:51:49 -07003332 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003333 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003334
asaperssonfab67072017-04-04 05:51:49 -07003335 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003336 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003337
asaperssonfab67072017-04-04 05:51:49 -07003338 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003339 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003340
kthelgason876222f2016-11-29 01:44:11 -08003341 // Expect a scale down.
3342 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003343 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003344
asapersson02465b82017-04-10 01:12:52 -07003345 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003346 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003347 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003348 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003349
asaperssonfab67072017-04-04 05:51:49 -07003350 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003351 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003352 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003353 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003354
asaperssonfab67072017-04-04 05:51:49 -07003355 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003356 EXPECT_EQ(std::numeric_limits<int>::max(),
3357 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003358
asaperssonfab67072017-04-04 05:51:49 -07003359 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003360 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003361 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003362 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003363
asapersson02465b82017-04-10 01:12:52 -07003364 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003365 EXPECT_EQ(std::numeric_limits<int>::max(),
3366 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003367
mflodmancc3d4422017-08-03 08:27:51 -07003368 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003369}
3370
mflodmancc3d4422017-08-03 08:27:51 -07003371TEST_F(VideoStreamEncoderTest,
3372 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003373 const int kWidth = 1280;
3374 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003375 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003376 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003377
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003378 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003379 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003380 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003381 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003382
3383 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003384 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003385 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003386 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3387 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3388
3389 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003390 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003391 EXPECT_THAT(source.sink_wants(),
3392 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003393 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3394 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3395 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3396
3397 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003398 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003399 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3400 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3401 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3402
mflodmancc3d4422017-08-03 08:27:51 -07003403 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003404}
3405
mflodmancc3d4422017-08-03 08:27:51 -07003406TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003407 const int kWidth = 1280;
3408 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003409 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003410 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003411
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003412 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003413 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003414 video_stream_encoder_->SetSource(&source,
3415 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003416 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3417 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003418 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003419
3420 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003421 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003422 EXPECT_THAT(source.sink_wants(),
3423 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003424 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3425 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3426 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3427
3428 // Trigger adapt down for same input resolution, expect no change.
3429 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3430 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003431 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003432 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3433 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3434 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3435
3436 // Trigger adapt down for larger input resolution, expect no change.
3437 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3438 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003439 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003440 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3441 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3442 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3443
mflodmancc3d4422017-08-03 08:27:51 -07003444 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003445}
3446
mflodmancc3d4422017-08-03 08:27:51 -07003447TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003448 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3449 const int kWidth = 640;
3450 const int kHeight = 360;
3451 const int64_t kFrameIntervalMs = 150;
3452 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003453 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003454
3455 // Enable BALANCED preference, no initial limitation.
3456 AdaptingFrameForwarder source(&time_controller_);
3457 source.set_adaptation_enabled(true);
3458 video_stream_encoder_->SetSource(&source,
3459 webrtc::DegradationPreference::BALANCED);
3460
3461 int64_t timestamp_ms = kFrameIntervalMs;
3462 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3463 sink_.WaitForEncodedFrame(kWidth, kHeight);
3464 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3465 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3466 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3467 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3468
3469 // Trigger adapt down, expect reduced fps (640x360@15fps).
3470 video_stream_encoder_->TriggerQualityLow();
3471 timestamp_ms += kFrameIntervalMs;
3472 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3473 sink_.WaitForEncodedFrame(timestamp_ms);
3474 EXPECT_THAT(source.sink_wants(),
3475 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3476 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3477 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3478 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3479
3480 // Source requests 270p, expect reduced resolution (480x270@15fps).
3481 source.OnOutputFormatRequest(480, 270);
3482 timestamp_ms += kFrameIntervalMs;
3483 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3484 WaitForEncodedFrame(480, 270);
3485 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3486
3487 // Trigger adapt down, expect reduced fps (480x270@10fps).
3488 video_stream_encoder_->TriggerQualityLow();
3489 timestamp_ms += kFrameIntervalMs;
3490 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3491 sink_.WaitForEncodedFrame(timestamp_ms);
3492 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3493 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3494 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3495 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3496
3497 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3498 source.OnOutputFormatRequest(320, 180);
3499 timestamp_ms += kFrameIntervalMs;
3500 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3501 WaitForEncodedFrame(320, 180);
3502 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3503
3504 // Trigger adapt down, expect reduced fps (320x180@7fps).
3505 video_stream_encoder_->TriggerQualityLow();
3506 timestamp_ms += kFrameIntervalMs;
3507 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3508 sink_.WaitForEncodedFrame(timestamp_ms);
3509 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3510 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3511 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3512 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3513
3514 // Source requests VGA, expect increased resolution (640x360@7fps).
3515 source.OnOutputFormatRequest(640, 360);
3516 timestamp_ms += kFrameIntervalMs;
3517 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3518 WaitForEncodedFrame(timestamp_ms);
3519 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3520
3521 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3522 video_stream_encoder_->TriggerQualityHigh();
3523 timestamp_ms += kFrameIntervalMs;
3524 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3525 WaitForEncodedFrame(timestamp_ms);
3526 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3527 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3528 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3529 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3530
3531 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3532 video_stream_encoder_->TriggerQualityHigh();
3533 timestamp_ms += kFrameIntervalMs;
3534 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3535 WaitForEncodedFrame(timestamp_ms);
3536 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3537 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3538 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3539 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3540
3541 // Trigger adapt up, expect increased fps (640x360@maxfps).
3542 video_stream_encoder_->TriggerQualityHigh();
3543 timestamp_ms += kFrameIntervalMs;
3544 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3545 WaitForEncodedFrame(timestamp_ms);
3546 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3547 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3548 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3549 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3550
3551 video_stream_encoder_->Stop();
3552}
3553
3554TEST_F(VideoStreamEncoderTest,
3555 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3556 const int kWidth = 1280;
3557 const int kHeight = 720;
3558 const int64_t kFrameIntervalMs = 150;
3559 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003560 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003561
3562 // Enable BALANCED preference, no initial limitation.
3563 AdaptingFrameForwarder source(&time_controller_);
3564 source.set_adaptation_enabled(true);
3565 video_stream_encoder_->SetSource(&source,
3566 webrtc::DegradationPreference::BALANCED);
3567
3568 int64_t timestamp_ms = kFrameIntervalMs;
3569 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3570 sink_.WaitForEncodedFrame(kWidth, kHeight);
3571 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3572 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3573 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3574 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3575
3576 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3577 video_stream_encoder_->TriggerQualityLow();
3578 timestamp_ms += kFrameIntervalMs;
3579 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3580 sink_.WaitForEncodedFrame(timestamp_ms);
3581 EXPECT_THAT(source.sink_wants(),
3582 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3583 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3584 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3585 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3586
3587 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3588 video_stream_encoder_->TriggerQualityLow();
3589 timestamp_ms += kFrameIntervalMs;
3590 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3591 sink_.WaitForEncodedFrame(timestamp_ms);
3592 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3593 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3594 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3595 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3596
3597 // Trigger adapt down, expect reduced fps (640x360@15fps).
3598 video_stream_encoder_->TriggerQualityLow();
3599 timestamp_ms += kFrameIntervalMs;
3600 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3601 WaitForEncodedFrame(timestamp_ms);
3602 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3603 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3604 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3605 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3606
3607 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3608 source.OnOutputFormatRequest(320, 180);
3609 timestamp_ms += kFrameIntervalMs;
3610 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3611 WaitForEncodedFrame(320, 180);
3612 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3613 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3614
3615 // Trigger adapt down, expect reduced fps (320x180@7fps).
3616 video_stream_encoder_->TriggerCpuOveruse();
3617 timestamp_ms += kFrameIntervalMs;
3618 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3619 WaitForEncodedFrame(timestamp_ms);
3620 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3621 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3622 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3623 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3624 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3625 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3626 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3627
3628 // Source requests HD, expect increased resolution (640x360@7fps).
3629 source.OnOutputFormatRequest(1280, 720);
3630 timestamp_ms += kFrameIntervalMs;
3631 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3632 WaitForEncodedFrame(timestamp_ms);
3633 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3634 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3635
3636 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3637 video_stream_encoder_->TriggerCpuUnderuse();
3638 timestamp_ms += kFrameIntervalMs;
3639 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3640 WaitForEncodedFrame(timestamp_ms);
3641 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3642 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3643 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3644 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3645 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3646 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3647 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3648
3649 // Trigger adapt up, expect increased fps (640x360@maxfps).
3650 video_stream_encoder_->TriggerQualityHigh();
3651 video_stream_encoder_->TriggerCpuUnderuse();
3652 timestamp_ms += kFrameIntervalMs;
3653 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3654 WaitForEncodedFrame(timestamp_ms);
3655 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3656 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3657 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3658 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3659 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3660 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3661 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3662
3663 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3664 video_stream_encoder_->TriggerQualityHigh();
3665 video_stream_encoder_->TriggerCpuUnderuse();
3666 timestamp_ms += kFrameIntervalMs;
3667 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3668 WaitForEncodedFrame(timestamp_ms);
3669 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3670 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3671 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3672 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3673 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3674 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3675 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3676
3677 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3678 video_stream_encoder_->TriggerQualityHigh();
3679 video_stream_encoder_->TriggerCpuUnderuse();
3680 timestamp_ms += kFrameIntervalMs;
3681 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3682 WaitForEncodedFrame(timestamp_ms);
3683 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3684 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3685 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3686 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3687 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3688 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3689 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3690
3691 video_stream_encoder_->Stop();
3692}
3693
3694TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003695 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003696 const int kWidth = 1280;
3697 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003698 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003699 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003700
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003701 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003702 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003703 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003704 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003705
3706 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003707 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003708 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003709 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3710 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3711
3712 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003713 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003714 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003715 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3716 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3717
mflodmancc3d4422017-08-03 08:27:51 -07003718 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003719}
3720
mflodmancc3d4422017-08-03 08:27:51 -07003721TEST_F(VideoStreamEncoderTest,
3722 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003723 const int kWidth = 1280;
3724 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003725 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003726 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003727
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003728 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003729 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003730 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003731 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003732
3733 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003734 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003735 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003736 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003737 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3738
3739 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003740 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003741 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003742 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003743 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3744
mflodmancc3d4422017-08-03 08:27:51 -07003745 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003746}
3747
mflodmancc3d4422017-08-03 08:27:51 -07003748TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003749 const int kWidth = 1280;
3750 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003751 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003752 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003753
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003754 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003755 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003756 video_stream_encoder_->SetSource(&source,
3757 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003758
3759 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3760 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003761 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003762 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3763 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3764 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3765
3766 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003767 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003768 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003769 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3770 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3771 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3772
mflodmancc3d4422017-08-03 08:27:51 -07003773 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003774}
3775
mflodmancc3d4422017-08-03 08:27:51 -07003776TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003777 const int kWidth = 1280;
3778 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003779 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003780 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003781
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003782 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003783 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003784 video_stream_encoder_->SetSource(&source,
3785 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003786
3787 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3788 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003789 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003790 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3791 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3792 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3793
3794 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003795 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003796 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003797 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3798 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3799 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3800
mflodmancc3d4422017-08-03 08:27:51 -07003801 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003802}
3803
mflodmancc3d4422017-08-03 08:27:51 -07003804TEST_F(VideoStreamEncoderTest,
3805 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003806 const int kWidth = 1280;
3807 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003808 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003809 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003810
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003811 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003812 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003813 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003814 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003815 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003816
3817 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003818 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003819 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003820 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3821 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3822
3823 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003824 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003825 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003826 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003827 EXPECT_THAT(source.sink_wants(),
3828 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003829 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3830 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3831
3832 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003833 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003834 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003835 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3836 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3837 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3838
mflodmancc3d4422017-08-03 08:27:51 -07003839 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003840}
3841
mflodmancc3d4422017-08-03 08:27:51 -07003842TEST_F(VideoStreamEncoderTest,
3843 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003844 const int kWidth = 1280;
3845 const int kHeight = 720;
3846 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003847 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003848 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003849
3850 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3851 stats.input_frame_rate = kInputFps;
3852 stats_proxy_->SetMockStats(stats);
3853
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003854 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003855 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3856 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003857 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003858
3859 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003860 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003861 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3862 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003863 EXPECT_THAT(video_source_.sink_wants(),
3864 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003865
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003866 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003867 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003868 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003869 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003870 // Give the encoder queue time to process the change in degradation preference
3871 // by waiting for an encoded frame.
3872 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3873 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003874 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003875
3876 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003877 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003878 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3879 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003880 EXPECT_THAT(new_video_source.sink_wants(),
3881 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003882
3883 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003884 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003885 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003886
mflodmancc3d4422017-08-03 08:27:51 -07003887 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003888}
3889
mflodmancc3d4422017-08-03 08:27:51 -07003890TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003891 const int kWidth = 1280;
3892 const int kHeight = 720;
3893 const size_t kNumFrames = 10;
3894
Henrik Boström381d1092020-05-12 18:49:07 +02003895 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003896 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003897
asaperssond0de2952017-04-21 01:47:31 -07003898 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003899 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003900 video_source_.set_adaptation_enabled(true);
3901
3902 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3903 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3904
3905 int downscales = 0;
3906 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003907 video_source_.IncomingCapturedFrame(
3908 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3909 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003910
asaperssonfab67072017-04-04 05:51:49 -07003911 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003912 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003913 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003914 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003915
3916 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3917 ++downscales;
3918
3919 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3920 EXPECT_EQ(downscales,
3921 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3922 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003923 }
mflodmancc3d4422017-08-03 08:27:51 -07003924 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003925}
3926
mflodmancc3d4422017-08-03 08:27:51 -07003927TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003928 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3929 const int kWidth = 1280;
3930 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003931 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003932 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003933
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003934 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003935 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003936 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003937 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003938 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003939
Åsa Persson8c1bf952018-09-13 10:42:19 +02003940 int64_t timestamp_ms = kFrameIntervalMs;
3941 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003942 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003943 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003944 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3945 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3946
3947 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003948 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003949 timestamp_ms += kFrameIntervalMs;
3950 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3951 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003952 EXPECT_THAT(source.sink_wants(),
3953 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003954 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3955 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3956
3957 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003958 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003959 timestamp_ms += kFrameIntervalMs;
3960 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003961 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003962 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003963 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3964 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3965
3966 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003967 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003968 timestamp_ms += kFrameIntervalMs;
3969 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3970 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003971 EXPECT_THAT(source.sink_wants(),
3972 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003973 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3974 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3975
3976 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003977 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003978 timestamp_ms += kFrameIntervalMs;
3979 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003980 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003981 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003982 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3983 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3984
mflodmancc3d4422017-08-03 08:27:51 -07003985 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003986}
3987
mflodmancc3d4422017-08-03 08:27:51 -07003988TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003989 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3990 const int kWidth = 1280;
3991 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003992 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003993 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003994
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003995 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003996 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003997 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003998 video_stream_encoder_->SetSource(&source,
3999 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07004000
Åsa Persson8c1bf952018-09-13 10:42:19 +02004001 int64_t timestamp_ms = kFrameIntervalMs;
4002 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07004003 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004004 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004005 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4006 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4007
4008 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07004009 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004010 timestamp_ms += kFrameIntervalMs;
4011 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4012 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004013 EXPECT_THAT(source.sink_wants(),
4014 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004015 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4016 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4017
4018 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07004019 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004020 timestamp_ms += kFrameIntervalMs;
4021 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07004022 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004023 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004024 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4025 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4026
4027 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07004028 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004029 timestamp_ms += kFrameIntervalMs;
4030 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4031 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004032 EXPECT_THAT(source.sink_wants(),
4033 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004034 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4035 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4036
4037 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07004038 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004039 timestamp_ms += kFrameIntervalMs;
4040 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07004041 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004042 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004043 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4044 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4045
mflodmancc3d4422017-08-03 08:27:51 -07004046 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004047}
4048
Sergey Silkin41c650b2019-10-14 13:12:19 +02004049TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
4050 fake_encoder_.SetResolutionBitrateLimits(
4051 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4052
Henrik Boström381d1092020-05-12 18:49:07 +02004053 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004054 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4055 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4056 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4057 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004058
4059 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004060 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004061 source.set_adaptation_enabled(true);
4062 video_stream_encoder_->SetSource(
4063 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4064
4065 // Insert 720p frame.
4066 int64_t timestamp_ms = kFrameIntervalMs;
4067 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4068 WaitForEncodedFrame(1280, 720);
4069
4070 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02004071 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004072 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4073 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4074 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4075 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004076 video_stream_encoder_->TriggerQualityLow();
4077
4078 // Insert 720p frame. It should be downscaled and encoded.
4079 timestamp_ms += kFrameIntervalMs;
4080 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4081 WaitForEncodedFrame(960, 540);
4082
4083 // Trigger adapt up. Higher resolution should not be requested duo to lack
4084 // of bitrate.
4085 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004086 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02004087
4088 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02004089 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004090 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4091 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4092 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4093 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004094
4095 // Trigger adapt up. Higher resolution should be requested.
4096 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004097 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02004098
4099 video_stream_encoder_->Stop();
4100}
4101
4102TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
4103 fake_encoder_.SetResolutionBitrateLimits(
4104 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4105
4106 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02004107 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004108 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4109 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4110 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4111 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004112
4113 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004114 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004115 source.set_adaptation_enabled(true);
4116 video_stream_encoder_->SetSource(
4117 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4118
4119 // Insert 720p frame. It should be dropped and lower resolution should be
4120 // requested.
4121 int64_t timestamp_ms = kFrameIntervalMs;
4122 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4123 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004124 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004125
4126 // Insert 720p frame. It should be downscaled and encoded.
4127 timestamp_ms += kFrameIntervalMs;
4128 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4129 WaitForEncodedFrame(960, 540);
4130
4131 video_stream_encoder_->Stop();
4132}
4133
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004134class BalancedDegradationTest : public VideoStreamEncoderTest {
4135 protected:
4136 void SetupTest() {
4137 // Reset encoder for field trials to take effect.
4138 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02004139 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004140
4141 // Enable BALANCED preference.
4142 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02004143 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4144 }
4145
Asa Persson606d3cb2021-10-04 10:07:11 +02004146 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02004147 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004148 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004149 }
4150
Åsa Persson45b176f2019-09-30 11:19:05 +02004151 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004152 timestamp_ms_ += kFrameIntervalMs;
4153 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02004154 }
4155
4156 void InsertFrameAndWaitForEncoded() {
4157 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004158 sink_.WaitForEncodedFrame(timestamp_ms_);
4159 }
4160
4161 const int kWidth = 640; // pixels:640x360=230400
4162 const int kHeight = 360;
4163 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4164 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004165 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004166};
4167
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004168TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004169 test::ScopedKeyValueConfig field_trials(
4170 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004171 "WebRTC-Video-BalancedDegradationSettings/"
4172 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4173 SetupTest();
4174
4175 // Force input frame rate.
4176 const int kInputFps = 24;
4177 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4178 stats.input_frame_rate = kInputFps;
4179 stats_proxy_->SetMockStats(stats);
4180
Åsa Persson45b176f2019-09-30 11:19:05 +02004181 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004182 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004183
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004184 // Trigger adapt down, expect scaled down framerate and resolution,
4185 // since Fps diff (input-requested:0) < threshold.
4186 video_stream_encoder_->TriggerQualityLow();
4187 EXPECT_THAT(source_.sink_wants(),
4188 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004189
4190 video_stream_encoder_->Stop();
4191}
4192
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004193TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004194 test::ScopedKeyValueConfig field_trials(
4195 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004196 "WebRTC-Video-BalancedDegradationSettings/"
4197 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4198 SetupTest();
4199
4200 // Force input frame rate.
4201 const int kInputFps = 25;
4202 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4203 stats.input_frame_rate = kInputFps;
4204 stats_proxy_->SetMockStats(stats);
4205
Åsa Persson45b176f2019-09-30 11:19:05 +02004206 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004207 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004208
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004209 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4210 // Fps diff (input-requested:1) == threshold.
4211 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004212 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004213
4214 video_stream_encoder_->Stop();
4215}
4216
4217TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004218 test::ScopedKeyValueConfig field_trials(
4219 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004220 "WebRTC-Video-BalancedDegradationSettings/"
4221 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4222 SetupTest();
4223
4224 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4225
Åsa Persson45b176f2019-09-30 11:19:05 +02004226 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004227 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004228
4229 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4230 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004231 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004232
4233 video_stream_encoder_->Stop();
4234}
4235
Åsa Perssonccfb3402019-09-25 15:13:04 +02004236TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004237 test::ScopedKeyValueConfig field_trials(
4238 field_trials_,
Åsa Persson1b247f12019-08-14 17:26:39 +02004239 "WebRTC-Video-BalancedDegradationSettings/"
4240 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004241 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004242
Asa Persson606d3cb2021-10-04 10:07:11 +02004243 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4244 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4245 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004246
Åsa Persson45b176f2019-09-30 11:19:05 +02004247 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004248 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004249 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4250
4251 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4252 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004253 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004254 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004255 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4256
4257 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4258 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004259 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004260 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004261 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4262
Åsa Persson30ab0152019-08-27 12:22:33 +02004263 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4264 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004265 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004266 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004267 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004268 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4269
4270 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004271 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004272 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004273 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004274
Åsa Persson30ab0152019-08-27 12:22:33 +02004275 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004276 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004277 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004278 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004279 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004280 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4281
4282 video_stream_encoder_->Stop();
4283}
4284
Åsa Perssonccfb3402019-09-25 15:13:04 +02004285TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004286 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004287 test::ScopedKeyValueConfig field_trials(
4288 field_trials_,
Åsa Persson45b176f2019-09-30 11:19:05 +02004289 "WebRTC-Video-BalancedDegradationSettings/"
4290 "pixels:57600|129600|230400,fps:7|24|24/");
4291 SetupTest();
Asa Persson606d3cb2021-10-04 10:07:11 +02004292 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 11:19:05 +02004293
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004294 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004295
4296 // Insert frame, expect scaled down:
4297 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4298 InsertFrame();
Markus Handell2cfc1af2022-08-19 08:16:48 +00004299 EXPECT_FALSE(WaitForFrame(TimeDelta::Seconds(1)));
Åsa Persson45b176f2019-09-30 11:19:05 +02004300 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4301 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4302
4303 // Insert frame, expect scaled down:
4304 // resolution (320x180@24fps).
4305 InsertFrame();
Markus Handell2cfc1af2022-08-19 08:16:48 +00004306 EXPECT_FALSE(WaitForFrame(TimeDelta::Seconds(1)));
Åsa Persson45b176f2019-09-30 11:19:05 +02004307 EXPECT_LT(source_.sink_wants().max_pixel_count,
4308 source_.last_wants().max_pixel_count);
4309 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4310
4311 // Frame should not be dropped (min pixels per frame reached).
4312 InsertFrameAndWaitForEncoded();
4313
4314 video_stream_encoder_->Stop();
4315}
4316
4317TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004318 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004319 test::ScopedKeyValueConfig field_trials(
4320 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004321 "WebRTC-Video-BalancedDegradationSettings/"
4322 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004323 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004324
Asa Persson606d3cb2021-10-04 10:07:11 +02004325 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4326 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4327 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004328
Åsa Persson45b176f2019-09-30 11:19:05 +02004329 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004330 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004331 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4332
4333 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4334 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004335 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004336 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004337 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4338
4339 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4340 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004341 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004342 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004343 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4344
4345 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4346 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004347 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004348 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004349 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4350
Åsa Persson30ab0152019-08-27 12:22:33 +02004351 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4352 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004353 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004354 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004355 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4356
4357 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4358 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004359 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004360 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4361
4362 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004363 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004364 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004365 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004366 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004367 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4368
4369 video_stream_encoder_->Stop();
4370}
4371
Åsa Perssonccfb3402019-09-25 15:13:04 +02004372TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004373 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004374 test::ScopedKeyValueConfig field_trials(
4375 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004376 "WebRTC-Video-BalancedDegradationSettings/"
4377 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004378 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004379
Asa Persson606d3cb2021-10-04 10:07:11 +02004380 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4381 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4382 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4383 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4384 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004385
Åsa Persson45b176f2019-09-30 11:19:05 +02004386 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004387 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004388 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4389
4390 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4391 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004392 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004393 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004394 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4395
4396 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4397 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004398 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004399 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004400 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4401
4402 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4403 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004404 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004405 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004406 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4407
4408 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4409 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004410 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004411 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4412
4413 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004414 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004415 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004416 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004417 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004418 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4419
4420 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004421 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004422 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004423 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004424 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4425
4426 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004427 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004428 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004429 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004430 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004431 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4432
Åsa Persson1b247f12019-08-14 17:26:39 +02004433 video_stream_encoder_->Stop();
4434}
4435
mflodmancc3d4422017-08-03 08:27:51 -07004436TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004437 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4438 const int kWidth = 1280;
4439 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004440 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004441 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004442
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004443 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004444 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004445 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004446 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004447 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004448
Åsa Persson8c1bf952018-09-13 10:42:19 +02004449 int64_t timestamp_ms = kFrameIntervalMs;
4450 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004451 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004452 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004453 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4454 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4455 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4456 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4457
4458 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004459 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004460 timestamp_ms += kFrameIntervalMs;
4461 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4462 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004463 EXPECT_THAT(source.sink_wants(),
4464 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004465 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4466 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4467 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4468 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4469
4470 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004471 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004472 timestamp_ms += kFrameIntervalMs;
4473 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4474 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004475 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004476 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4477 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4478 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4479 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4480
Jonathan Yubc771b72017-12-08 17:04:29 -08004481 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004482 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004483 timestamp_ms += kFrameIntervalMs;
4484 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4485 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004486 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004487 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4488 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004489 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004490 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4491
Jonathan Yubc771b72017-12-08 17:04:29 -08004492 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004493 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004494 timestamp_ms += kFrameIntervalMs;
4495 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4496 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004497 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004498 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004499 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4500 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4501 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4502 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4503
Jonathan Yubc771b72017-12-08 17:04:29 -08004504 // Trigger quality adapt down, expect no change (min resolution reached).
4505 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004506 timestamp_ms += kFrameIntervalMs;
4507 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4508 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004509 EXPECT_THAT(source.sink_wants(), FpsMax());
4510 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004511 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4512 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4513 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4514 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4515
Evan Shrubsole64469032020-06-11 10:45:29 +02004516 // Trigger quality adapt up, expect upscaled resolution (480x270).
4517 video_stream_encoder_->TriggerQualityHigh();
4518 timestamp_ms += kFrameIntervalMs;
4519 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4520 WaitForEncodedFrame(timestamp_ms);
4521 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4522 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4523 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4524 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4525 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4526
4527 // Trigger quality and cpu adapt up since both are most limited, expect
4528 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004529 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004530 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004531 timestamp_ms += kFrameIntervalMs;
4532 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4533 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004534 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004535 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4536 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4537 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004538 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004539
Evan Shrubsole64469032020-06-11 10:45:29 +02004540 // Trigger quality and cpu adapt up since both are most limited, expect
4541 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004542 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004543 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004544 timestamp_ms += kFrameIntervalMs;
4545 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4546 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004547 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004548 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004549 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004550 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004551 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4552 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004553
Evan Shrubsole64469032020-06-11 10:45:29 +02004554 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4555 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004556 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004557 timestamp_ms += kFrameIntervalMs;
4558 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4559 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004560 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004561 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4562 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004563 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004564 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004565
4566 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004567 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004568 timestamp_ms += kFrameIntervalMs;
4569 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004570 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004571 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004572 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004573 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4574 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004575 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004576 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004577
mflodmancc3d4422017-08-03 08:27:51 -07004578 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004579}
4580
mflodmancc3d4422017-08-03 08:27:51 -07004581TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004582 const int kWidth = 640;
4583 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004584
Henrik Boström381d1092020-05-12 18:49:07 +02004585 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004586 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004587
perkj803d97f2016-11-01 11:45:46 -07004588 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004589 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004590 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004591 }
4592
mflodmancc3d4422017-08-03 08:27:51 -07004593 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004594 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004595 video_source_.IncomingCapturedFrame(CreateFrame(
4596 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004597 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004598 }
4599
mflodmancc3d4422017-08-03 08:27:51 -07004600 video_stream_encoder_->Stop();
4601 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004602 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004603
Ying Wangef3998f2019-12-09 13:06:53 +01004604 EXPECT_METRIC_EQ(
4605 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4606 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004607 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4608}
4609
mflodmancc3d4422017-08-03 08:27:51 -07004610TEST_F(VideoStreamEncoderTest,
4611 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004612 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004613 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004614 const int kWidth = 640;
4615 const int kHeight = 360;
4616
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004617 video_stream_encoder_->SetSource(&video_source_,
4618 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004619
4620 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4621 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004622 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004623 }
4624
mflodmancc3d4422017-08-03 08:27:51 -07004625 video_stream_encoder_->Stop();
4626 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004627 stats_proxy_.reset();
4628
4629 EXPECT_EQ(0,
4630 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4631}
4632
Per Kjellanderdcef6412020-10-07 15:09:05 +02004633TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4634 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004635 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004636 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004637
4638 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004639 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 10:07:11 +02004640 SimulcastRateAllocator(fake_encoder_.config())
4641 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004642 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004643
Henrik Boström381d1092020-05-12 18:49:07 +02004644 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004645 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004646
sprang57c2fff2017-01-16 06:24:02 -08004647 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004648 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4649 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004650 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4651 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4652
Erik Språngd7329ca2019-02-21 21:19:53 +01004653 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004654 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004655 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004656
Per Kjellanderdcef6412020-10-07 15:09:05 +02004657 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004658 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004659 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4660 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004661 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004662 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004663
Per Kjellanderdcef6412020-10-07 15:09:05 +02004664 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004665 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004666 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004667 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004668 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4669 WaitForEncodedFrame(CurrentTimeMs());
4670 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004671 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004672 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004673
mflodmancc3d4422017-08-03 08:27:51 -07004674 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004675}
4676
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004677TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004678 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004679 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004680 kVideoLayersAllocation);
4681
4682 const int kDefaultFps = 30;
4683
4684 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004685 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004686
4687 video_source_.IncomingCapturedFrame(
4688 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4689 WaitForEncodedFrame(CurrentTimeMs());
4690 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4691 VideoLayersAllocation last_layer_allocation =
4692 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004693 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004694 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4695
4696 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004697 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004698 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 10:07:11 +02004699 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 17:53:22 +02004700 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4701
Erik Språng9d69cbe2020-10-22 17:44:42 +02004702 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004703 int number_of_layers_allocation = 1;
4704 const int64_t start_time_ms = CurrentTimeMs();
4705 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4706 video_source_.IncomingCapturedFrame(
4707 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4708 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004709 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4710 number_of_layers_allocation = sink_.number_of_layers_allocations();
4711 VideoLayersAllocation new_allocation =
4712 sink_.GetLastVideoLayersAllocation();
4713 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4714 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4715 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4716 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4717 .target_bitrate_per_temporal_layer,
4718 last_layer_allocation.active_spatial_layers[0]
4719 .target_bitrate_per_temporal_layer);
4720 last_layer_allocation = new_allocation;
4721 }
4722 }
4723 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4724 video_stream_encoder_->Stop();
4725}
4726
4727TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004728 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004729 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4730 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4731 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004732 VideoEncoderConfig video_encoder_config;
4733 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4734 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004735 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004736 video_encoder_config.content_type =
4737 VideoEncoderConfig::ContentType::kRealtimeVideo;
4738 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004739 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004740 VideoEncoder::GetDefaultVp8Settings());
4741 for (auto& layer : video_encoder_config.simulcast_layers) {
4742 layer.num_temporal_layers = 2;
4743 }
4744 // Simulcast layers are used for enabling/disabling streams.
4745 video_encoder_config.simulcast_layers[0].active = true;
4746 video_encoder_config.simulcast_layers[1].active = false;
4747 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004748 ConfigureEncoder(std::move(video_encoder_config),
4749 VideoStreamEncoder::BitrateAllocationCallbackType::
4750 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004751
4752 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004753 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004754
4755 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4756 WaitForEncodedFrame(CurrentTimeMs());
4757 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4758 VideoLayersAllocation last_layer_allocation =
4759 sink_.GetLastVideoLayersAllocation();
4760
4761 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4762 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4763 .target_bitrate_per_temporal_layer,
4764 SizeIs(2));
4765 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4766 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4767 video_stream_encoder_->Stop();
4768}
4769
4770TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004771 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004772 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4773 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4774 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004775 VideoEncoderConfig video_encoder_config;
4776 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4777 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004778 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004779 video_encoder_config.content_type =
4780 VideoEncoderConfig::ContentType::kRealtimeVideo;
4781 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004782 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004783 VideoEncoder::GetDefaultVp8Settings());
4784 for (auto& layer : video_encoder_config.simulcast_layers) {
4785 layer.num_temporal_layers = 2;
4786 }
4787 // Simulcast layers are used for enabling/disabling streams.
4788 video_encoder_config.simulcast_layers[0].active = true;
4789 video_encoder_config.simulcast_layers[1].active = false;
4790 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004791 ConfigureEncoder(std::move(video_encoder_config),
4792 VideoStreamEncoder::BitrateAllocationCallbackType::
4793 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004794
4795 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004796 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004797
4798 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4799 WaitForEncodedFrame(CurrentTimeMs());
4800 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4801 VideoLayersAllocation last_layer_allocation =
4802 sink_.GetLastVideoLayersAllocation();
4803
4804 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4805 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4806 .target_bitrate_per_temporal_layer,
4807 SizeIs(2));
4808 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4809
4810 video_stream_encoder_->Stop();
4811}
4812
4813TEST_F(VideoStreamEncoderTest,
4814 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4815 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4816 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004817 VideoEncoderConfig video_encoder_config;
4818 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4819 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004820 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004821 video_encoder_config.content_type =
4822 VideoEncoderConfig::ContentType::kRealtimeVideo;
4823 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4824 vp9_settings.numberOfSpatialLayers = 2;
4825 vp9_settings.numberOfTemporalLayers = 2;
4826 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4827 vp9_settings.automaticResizeOn = false;
4828 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004829 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004830 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004831 ConfigureEncoder(std::move(video_encoder_config),
4832 VideoStreamEncoder::BitrateAllocationCallbackType::
4833 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004834
4835 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004836 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004837
4838 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4839 WaitForEncodedFrame(CurrentTimeMs());
4840 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4841 VideoLayersAllocation last_layer_allocation =
4842 sink_.GetLastVideoLayersAllocation();
4843
4844 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4845 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4846 .target_bitrate_per_temporal_layer,
4847 SizeIs(2));
4848 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4849 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4850 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4851 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4852 .target_bitrate_per_temporal_layer,
4853 SizeIs(2));
4854 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4855 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4856 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4857
4858 // Since full SVC is used, expect the top layer to utilize the full target
4859 // rate.
4860 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4861 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004862 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004863 video_stream_encoder_->Stop();
4864}
4865
4866TEST_F(VideoStreamEncoderTest,
4867 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4868 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4869 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004870 VideoEncoderConfig video_encoder_config;
4871 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4872 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004873 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004874 video_encoder_config.content_type =
4875 VideoEncoderConfig::ContentType::kRealtimeVideo;
4876 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4877 vp9_settings.numberOfSpatialLayers = 2;
4878 vp9_settings.numberOfTemporalLayers = 2;
4879 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4880 vp9_settings.automaticResizeOn = false;
4881 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004882 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004883 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004884 ConfigureEncoder(std::move(video_encoder_config),
4885 VideoStreamEncoder::BitrateAllocationCallbackType::
4886 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004887
4888 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004889 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004890
4891 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4892 WaitForEncodedFrame(CurrentTimeMs());
4893 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4894 VideoLayersAllocation last_layer_allocation =
4895 sink_.GetLastVideoLayersAllocation();
4896
4897 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4898 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4899 .target_bitrate_per_temporal_layer,
4900 SizeIs(1));
4901 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4902 .target_bitrate_per_temporal_layer,
4903 SizeIs(1));
4904 // Since full SVC is used, expect the top layer to utilize the full target
4905 // rate.
4906 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4907 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004908 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004909 video_stream_encoder_->Stop();
4910}
4911
4912TEST_F(VideoStreamEncoderTest,
4913 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4914 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4915 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004916 VideoEncoderConfig video_encoder_config;
4917 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4918 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004919 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004920 video_encoder_config.content_type =
4921 VideoEncoderConfig::ContentType::kRealtimeVideo;
4922 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4923 vp9_settings.numberOfSpatialLayers = 2;
4924 vp9_settings.numberOfTemporalLayers = 2;
4925 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4926 vp9_settings.automaticResizeOn = false;
4927 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004928 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004929 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004930 ConfigureEncoder(std::move(video_encoder_config),
4931 VideoStreamEncoder::BitrateAllocationCallbackType::
4932 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004933
4934 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004935 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004936
4937 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4938 WaitForEncodedFrame(CurrentTimeMs());
4939 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4940 VideoLayersAllocation last_layer_allocation =
4941 sink_.GetLastVideoLayersAllocation();
4942
4943 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4944 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4945 .target_bitrate_per_temporal_layer,
4946 SizeIs(2));
4947 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4948 .target_bitrate_per_temporal_layer,
4949 SizeIs(2));
4950 // Since KSVC is, spatial layers are independend except on key frames.
4951 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4952 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004953 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004954 video_stream_encoder_->Stop();
4955}
4956
4957TEST_F(VideoStreamEncoderTest,
4958 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4959 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4960 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4961 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004962 VideoEncoderConfig video_encoder_config;
4963 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4964 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004965 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004966 video_encoder_config.content_type =
4967 VideoEncoderConfig::ContentType::kRealtimeVideo;
4968 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4969 vp9_settings.numberOfSpatialLayers = 3;
4970 vp9_settings.numberOfTemporalLayers = 2;
4971 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4972 vp9_settings.automaticResizeOn = false;
4973 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004974 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004975 vp9_settings);
4976 // Simulcast layers are used for enabling/disabling streams.
4977 video_encoder_config.simulcast_layers.resize(3);
4978 video_encoder_config.simulcast_layers[0].active = false;
4979 video_encoder_config.simulcast_layers[1].active = true;
4980 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004981 ConfigureEncoder(std::move(video_encoder_config),
4982 VideoStreamEncoder::BitrateAllocationCallbackType::
4983 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004984
4985 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004986 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004987
4988 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4989 WaitForEncodedFrame(CurrentTimeMs());
4990 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4991 VideoLayersAllocation last_layer_allocation =
4992 sink_.GetLastVideoLayersAllocation();
4993
4994 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4995 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4996 .target_bitrate_per_temporal_layer,
4997 SizeIs(2));
4998 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4999 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5000
5001 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
5002 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
5003 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
5004 .target_bitrate_per_temporal_layer,
5005 SizeIs(2));
5006 // Since full SVC is used, expect the top layer to utilize the full target
5007 // rate.
5008 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
5009 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02005010 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005011 video_stream_encoder_->Stop();
5012}
5013
5014TEST_F(VideoStreamEncoderTest,
5015 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
5016 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5017 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5018 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005019 VideoEncoderConfig video_encoder_config;
5020 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5021 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005022 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005023 video_encoder_config.content_type =
5024 VideoEncoderConfig::ContentType::kRealtimeVideo;
5025 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5026 vp9_settings.numberOfSpatialLayers = 3;
5027 vp9_settings.numberOfTemporalLayers = 2;
5028 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5029 vp9_settings.automaticResizeOn = false;
5030 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005031 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005032 vp9_settings);
5033 // Simulcast layers are used for enabling/disabling streams.
5034 video_encoder_config.simulcast_layers.resize(3);
5035 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005036 ConfigureEncoder(std::move(video_encoder_config),
5037 VideoStreamEncoder::BitrateAllocationCallbackType::
5038 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005039
5040 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005041 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005042
5043 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5044 WaitForEncodedFrame(CurrentTimeMs());
5045 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5046 VideoLayersAllocation last_layer_allocation =
5047 sink_.GetLastVideoLayersAllocation();
5048
5049 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
5050 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5051 .target_bitrate_per_temporal_layer,
5052 SizeIs(2));
5053 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
5054 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5055
5056 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
5057 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
5058 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
5059 .target_bitrate_per_temporal_layer,
5060 SizeIs(2));
5061 video_stream_encoder_->Stop();
5062}
5063
5064TEST_F(VideoStreamEncoderTest,
5065 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
5066 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5067 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5068 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005069 VideoEncoderConfig video_encoder_config;
5070 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5071 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005072 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005073 video_encoder_config.content_type =
5074 VideoEncoderConfig::ContentType::kRealtimeVideo;
5075 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5076 vp9_settings.numberOfSpatialLayers = 3;
5077 vp9_settings.numberOfTemporalLayers = 2;
5078 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5079 vp9_settings.automaticResizeOn = false;
5080 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005081 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005082 vp9_settings);
5083 // Simulcast layers are used for enabling/disabling streams.
5084 video_encoder_config.simulcast_layers.resize(3);
5085 video_encoder_config.simulcast_layers[0].active = false;
5086 video_encoder_config.simulcast_layers[1].active = false;
5087 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005088 ConfigureEncoder(std::move(video_encoder_config),
5089 VideoStreamEncoder::BitrateAllocationCallbackType::
5090 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005091
5092 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005093 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005094
5095 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5096 WaitForEncodedFrame(CurrentTimeMs());
5097 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5098 VideoLayersAllocation last_layer_allocation =
5099 sink_.GetLastVideoLayersAllocation();
5100
5101 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5102 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5103 .target_bitrate_per_temporal_layer,
5104 SizeIs(2));
5105 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5106 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5107 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5108 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02005109 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005110 video_stream_encoder_->Stop();
5111}
5112
5113TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5114 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005115 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005116 kVideoLayersAllocation);
5117 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005118 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005119
5120 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5121 WaitForEncodedFrame(CurrentTimeMs());
5122 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5123 VideoLayersAllocation last_layer_allocation =
5124 sink_.GetLastVideoLayersAllocation();
5125
5126 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5127 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5128 .target_bitrate_per_temporal_layer,
5129 SizeIs(1));
5130 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5131 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005132 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005133 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5134 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5135 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5136 video_stream_encoder_->Stop();
5137}
5138
5139TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02005140 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5141 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005142 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02005143 kVideoLayersAllocation);
5144
5145 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005146 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005147
5148 video_source_.IncomingCapturedFrame(
5149 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5150 WaitForEncodedFrame(CurrentTimeMs());
5151 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5152 VideoLayersAllocation last_layer_allocation =
5153 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02005154 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02005155 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5156 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5157 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005158 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 17:53:22 +02005159
5160 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005161 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5162 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005163 video_source_.IncomingCapturedFrame(
5164 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5165 WaitForEncodedFrame(CurrentTimeMs());
5166
5167 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5168 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5169 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5170 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5171 .target_bitrate_per_temporal_layer[0],
5172 DataRate::Zero());
5173
5174 video_stream_encoder_->Stop();
5175}
5176
Per Kjellander4190ce92020-12-15 17:24:55 +01005177TEST_F(VideoStreamEncoderTest,
5178 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5179 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005180 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01005181 kVideoLayersAllocation);
5182
5183 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005184 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5185 0, 0, 0);
Per Kjellander4190ce92020-12-15 17:24:55 +01005186
5187 video_source_.IncomingCapturedFrame(
5188 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5189 WaitForEncodedFrame(CurrentTimeMs());
5190 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5191 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5192 SizeIs(2));
5193 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5194 codec_width_);
5195 EXPECT_EQ(
5196 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5197 codec_height_);
5198
5199 video_source_.IncomingCapturedFrame(
5200 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5201 WaitForEncodedFrame(CurrentTimeMs());
5202 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5203 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5204 SizeIs(2));
5205 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5206 codec_width_ / 2);
5207 EXPECT_EQ(
5208 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5209 codec_height_ / 2);
5210
5211 video_stream_encoder_->Stop();
5212}
5213
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005214TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5215 // 2 TLs configured, temporal layers supported by encoder.
5216 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005217 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005218 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005219 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005220 fake_encoder_.SetTemporalLayersSupported(0, true);
5221
5222 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 10:07:11 +02005223 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005224 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005225 kNumTemporalLayers, /*temporal_id*/ 0,
5226 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005227 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005228 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005229 kNumTemporalLayers, /*temporal_id*/ 1,
5230 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005231 VideoBitrateAllocation expected_bitrate;
5232 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5233 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5234
5235 VerifyAllocatedBitrate(expected_bitrate);
5236 video_stream_encoder_->Stop();
5237}
5238
5239TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5240 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005241 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005242 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005243 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005244 fake_encoder_.SetTemporalLayersSupported(0, false);
5245
5246 // Temporal layers not supported by the encoder.
5247 // Total bitrate should be at ti:0.
5248 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005249 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005250
5251 VerifyAllocatedBitrate(expected_bitrate);
5252 video_stream_encoder_->Stop();
5253}
5254
5255TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005256 webrtc::test::ScopedKeyValueConfig field_trials(
5257 field_trials_,
Per Kjellanderdcef6412020-10-07 15:09:05 +02005258 "WebRTC-Video-QualityScalerSettings/"
5259 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5260 // Reset encoder for field trials to take effect.
5261 ConfigureEncoder(video_encoder_config_.Copy());
5262
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005263 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005264 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005265 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005266 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005267 fake_encoder_.SetTemporalLayersSupported(0, true);
5268 fake_encoder_.SetTemporalLayersSupported(1, false);
5269
5270 const int kS0Bps = 150000;
5271 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005272 kS0Bps *
5273 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5274 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005275 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005276 kS0Bps *
5277 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5278 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005279 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005280 // Temporal layers not supported by si:1.
5281 VideoBitrateAllocation expected_bitrate;
5282 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5283 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5284 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5285
5286 VerifyAllocatedBitrate(expected_bitrate);
5287 video_stream_encoder_->Stop();
5288}
5289
Niels Möller7dc26b72017-12-06 10:27:48 +01005290TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5291 const int kFrameWidth = 1280;
5292 const int kFrameHeight = 720;
5293 const int kFramerate = 24;
5294
Henrik Boström381d1092020-05-12 18:49:07 +02005295 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005296 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005297 test::FrameForwarder source;
5298 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005299 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005300
5301 // Insert a single frame, triggering initial configuration.
5302 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5303 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5304
5305 EXPECT_EQ(
5306 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5307 kDefaultFramerate);
5308
5309 // Trigger reconfigure encoder (without resetting the entire instance).
5310 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005311 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5312 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005313 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005314 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005315 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005316 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5317
5318 // Detector should be updated with fps limit from codec config.
5319 EXPECT_EQ(
5320 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5321 kFramerate);
5322
5323 // Trigger overuse, max framerate should be reduced.
5324 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5325 stats.input_frame_rate = kFramerate;
5326 stats_proxy_->SetMockStats(stats);
5327 video_stream_encoder_->TriggerCpuOveruse();
5328 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5329 int adapted_framerate =
5330 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5331 EXPECT_LT(adapted_framerate, kFramerate);
5332
5333 // Trigger underuse, max framerate should go back to codec configured fps.
5334 // Set extra low fps, to make sure it's actually reset, not just incremented.
5335 stats = stats_proxy_->GetStats();
5336 stats.input_frame_rate = adapted_framerate / 2;
5337 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005338 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005339 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5340 EXPECT_EQ(
5341 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5342 kFramerate);
5343
5344 video_stream_encoder_->Stop();
5345}
5346
5347TEST_F(VideoStreamEncoderTest,
5348 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5349 const int kFrameWidth = 1280;
5350 const int kFrameHeight = 720;
5351 const int kLowFramerate = 15;
5352 const int kHighFramerate = 25;
5353
Henrik Boström381d1092020-05-12 18:49:07 +02005354 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005355 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005356 test::FrameForwarder source;
5357 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005358 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005359
5360 // Trigger initial configuration.
5361 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005362 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5363 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005364 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005365 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005366 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005367 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005368 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5369
5370 EXPECT_EQ(
5371 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5372 kLowFramerate);
5373
5374 // Trigger overuse, max framerate should be reduced.
5375 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5376 stats.input_frame_rate = kLowFramerate;
5377 stats_proxy_->SetMockStats(stats);
5378 video_stream_encoder_->TriggerCpuOveruse();
5379 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5380 int adapted_framerate =
5381 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5382 EXPECT_LT(adapted_framerate, kLowFramerate);
5383
5384 // Reconfigure the encoder with a new (higher max framerate), max fps should
5385 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005386 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005387 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5388 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005389 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005390 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5391
5392 EXPECT_EQ(
5393 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5394 adapted_framerate);
5395
5396 // Trigger underuse, max framerate should go back to codec configured fps.
5397 stats = stats_proxy_->GetStats();
5398 stats.input_frame_rate = adapted_framerate;
5399 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005400 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005401 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5402 EXPECT_EQ(
5403 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5404 kHighFramerate);
5405
5406 video_stream_encoder_->Stop();
5407}
5408
mflodmancc3d4422017-08-03 08:27:51 -07005409TEST_F(VideoStreamEncoderTest,
5410 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005411 const int kFrameWidth = 1280;
5412 const int kFrameHeight = 720;
5413 const int kFramerate = 24;
5414
Henrik Boström381d1092020-05-12 18:49:07 +02005415 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005416 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005417 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005418 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005419 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005420
5421 // Trigger initial configuration.
5422 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005423 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5424 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005425 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 04:21:07 -07005426 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005427 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005428 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005429 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005430
Niels Möller7dc26b72017-12-06 10:27:48 +01005431 EXPECT_EQ(
5432 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5433 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005434
5435 // Trigger overuse, max framerate should be reduced.
5436 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5437 stats.input_frame_rate = kFramerate;
5438 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005439 video_stream_encoder_->TriggerCpuOveruse();
5440 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005441 int adapted_framerate =
5442 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005443 EXPECT_LT(adapted_framerate, kFramerate);
5444
5445 // Change degradation preference to not enable framerate scaling. Target
5446 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005447 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005448 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005449 EXPECT_EQ(
5450 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5451 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005452
mflodmancc3d4422017-08-03 08:27:51 -07005453 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005454}
5455
mflodmancc3d4422017-08-03 08:27:51 -07005456TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005457 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005458 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005459 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5460 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5461 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005462 const int kWidth = 640;
5463 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005464
asaperssonfab67072017-04-04 05:51:49 -07005465 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005466
5467 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005468 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005469
5470 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005471 EXPECT_TRUE_WAIT(
5472 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005473
sprangc5d62e22017-04-02 23:53:04 -07005474 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005475
asaperssonfab67072017-04-04 05:51:49 -07005476 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005477 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005478 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005479
5480 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005481 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005482
Henrik Boström2671dac2020-05-19 16:29:09 +02005483 EXPECT_TRUE_WAIT(
5484 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005485
mflodmancc3d4422017-08-03 08:27:51 -07005486 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005487}
5488
mflodmancc3d4422017-08-03 08:27:51 -07005489TEST_F(VideoStreamEncoderTest,
5490 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005491 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005492 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005493 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5494 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5495 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005496 const int kWidth = 640;
5497 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005498
5499 // We expect the n initial frames to get dropped.
5500 int i;
5501 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005502 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005503 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005504 }
5505 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005506 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005507 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005508
5509 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005510 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005511
mflodmancc3d4422017-08-03 08:27:51 -07005512 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005513}
5514
mflodmancc3d4422017-08-03 08:27:51 -07005515TEST_F(VideoStreamEncoderTest,
5516 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005517 const int kWidth = 640;
5518 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005519 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005520 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005521
5522 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005523 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005524 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005525
asaperssonfab67072017-04-04 05:51:49 -07005526 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005527 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005528 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005529
mflodmancc3d4422017-08-03 08:27:51 -07005530 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005531}
5532
mflodmancc3d4422017-08-03 08:27:51 -07005533TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005534 const int kWidth = 640;
5535 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005536 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005537
5538 VideoEncoderConfig video_encoder_config;
5539 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5540 // Make format different, to force recreation of encoder.
5541 video_encoder_config.video_format.parameters["foo"] = "foo";
5542 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005543 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005544 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005545 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005546
kthelgasonb83797b2017-02-14 11:57:25 -08005547 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005548 video_stream_encoder_->SetSource(&video_source_,
5549 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005550
asaperssonfab67072017-04-04 05:51:49 -07005551 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005552 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005553 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005554
mflodmancc3d4422017-08-03 08:27:51 -07005555 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005556 fake_encoder_.SetQualityScaling(true);
5557}
5558
Åsa Persson139f4dc2019-08-02 09:29:58 +02005559TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005560 webrtc::test::ScopedKeyValueConfig field_trials(
5561 field_trials_,
Åsa Persson139f4dc2019-08-02 09:29:58 +02005562 "WebRTC-Video-QualityScalerSettings/"
5563 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5564 // Reset encoder for field trials to take effect.
5565 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005566 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5567 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 09:29:58 +02005568 const int kWidth = 640;
5569 const int kHeight = 360;
5570
Henrik Boström381d1092020-05-12 18:49:07 +02005571 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005572 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005573 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5574 // Frame should not be dropped.
5575 WaitForEncodedFrame(1);
5576
Henrik Boström381d1092020-05-12 18:49:07 +02005577 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005578 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5579 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5580 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005581 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5582 // Frame should not be dropped.
5583 WaitForEncodedFrame(2);
5584
Henrik Boström381d1092020-05-12 18:49:07 +02005585 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005586 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5587 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5588 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005589 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5590 // Expect to drop this frame, the wait should time out.
5591 ExpectDroppedFrame();
5592
5593 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005594 EXPECT_TRUE_WAIT(
5595 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005596 video_stream_encoder_->Stop();
5597}
5598
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005599TEST_F(VideoStreamEncoderTest,
5600 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005601 webrtc::test::ScopedKeyValueConfig field_trials(
5602 field_trials_,
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005603 "WebRTC-Video-QualityScalerSettings/"
5604 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5605 fake_encoder_.SetQualityScaling(false);
5606 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005607 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5608 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005609 const int kWidth = 640;
5610 const int kHeight = 360;
5611
5612 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005613 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005614 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5615 // Frame should not be dropped.
5616 WaitForEncodedFrame(1);
5617
5618 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5619 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5620 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5621 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5622 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5623 // Frame should not be dropped.
5624 WaitForEncodedFrame(2);
5625
5626 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5627 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5628 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5629 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5630 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5631 // Not dropped since quality scaling is disabled.
5632 WaitForEncodedFrame(3);
5633
5634 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005635 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005636 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5637
5638 video_stream_encoder_->Stop();
5639}
5640
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005641TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005642 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005643 // Set simulcast.
5644 ResetEncoder("VP8", 3, 1, 1, false);
5645 fake_encoder_.SetQualityScaling(true);
5646 const int kWidth = 1280;
5647 const int kHeight = 720;
5648 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005649 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005650 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5651 // Frame should not be dropped.
5652 WaitForEncodedFrame(1);
5653
5654 // Trigger QVGA "singlecast"
5655 // Update the config.
5656 VideoEncoderConfig video_encoder_config;
Jonas Oreland80c87d72022-09-29 15:01:09 +02005657 webrtc::VideoEncoder::EncoderInfo encoder_info;
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005658 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5659 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005660 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005661 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005662 "VP8", /*max qp*/ 56, /*screencast*/ false,
Jonas Oreland80c87d72022-09-29 15:01:09 +02005663 /*screenshare enabled*/ false, encoder_info);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005664 for (auto& layer : video_encoder_config.simulcast_layers) {
5665 layer.num_temporal_layers = 1;
5666 layer.max_framerate = kDefaultFramerate;
5667 }
Asa Persson606d3cb2021-10-04 10:07:11 +02005668 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005669 video_encoder_config.content_type =
5670 VideoEncoderConfig::ContentType::kRealtimeVideo;
5671
5672 video_encoder_config.simulcast_layers[0].active = true;
5673 video_encoder_config.simulcast_layers[1].active = false;
5674 video_encoder_config.simulcast_layers[2].active = false;
5675
5676 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5677 kMaxPayloadLength);
5678 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5679
5680 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5681 // Frame should not be dropped.
5682 WaitForEncodedFrame(2);
5683
5684 // Trigger HD "singlecast"
5685 video_encoder_config.simulcast_layers[0].active = false;
5686 video_encoder_config.simulcast_layers[1].active = false;
5687 video_encoder_config.simulcast_layers[2].active = true;
5688
5689 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5690 kMaxPayloadLength);
5691 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5692
5693 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5694 // Frame should be dropped because of initial frame drop.
5695 ExpectDroppedFrame();
5696
5697 // Expect the sink_wants to specify a scaled frame.
5698 EXPECT_TRUE_WAIT(
5699 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5700 video_stream_encoder_->Stop();
5701}
5702
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005703TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005704 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005705 // Set simulcast.
5706 ResetEncoder("VP9", 1, 1, 3, false);
5707 fake_encoder_.SetQualityScaling(true);
5708 const int kWidth = 1280;
5709 const int kHeight = 720;
5710 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005711 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005712 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5713 // Frame should not be dropped.
5714 WaitForEncodedFrame(1);
5715
5716 // Trigger QVGA "singlecast"
5717 // Update the config.
5718 VideoEncoderConfig video_encoder_config;
5719 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5720 &video_encoder_config);
5721 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5722 vp9_settings.numberOfSpatialLayers = 3;
5723 // Since only one layer is active - automatic resize should be enabled.
5724 vp9_settings.automaticResizeOn = true;
5725 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005726 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005727 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005728 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005729 video_encoder_config.content_type =
5730 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 16:22:11 +02005731 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005732 // which SVC layers are active.
5733 video_encoder_config.simulcast_layers.resize(3);
5734
5735 video_encoder_config.simulcast_layers[0].active = true;
5736 video_encoder_config.simulcast_layers[1].active = false;
5737 video_encoder_config.simulcast_layers[2].active = false;
5738
5739 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5740 kMaxPayloadLength);
5741 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5742
5743 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5744 // Frame should not be dropped.
5745 WaitForEncodedFrame(2);
5746
5747 // Trigger HD "singlecast"
5748 video_encoder_config.simulcast_layers[0].active = false;
5749 video_encoder_config.simulcast_layers[1].active = false;
5750 video_encoder_config.simulcast_layers[2].active = true;
5751
5752 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5753 kMaxPayloadLength);
5754 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5755
5756 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5757 // Frame should be dropped because of initial frame drop.
5758 ExpectDroppedFrame();
5759
5760 // Expect the sink_wants to specify a scaled frame.
5761 EXPECT_TRUE_WAIT(
5762 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5763 video_stream_encoder_->Stop();
5764}
5765
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005766TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005767 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5768 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5769 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5770 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5771 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5772 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5773 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5774 fake_encoder_.SetResolutionBitrateLimits(
5775 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5776
5777 VideoEncoderConfig video_encoder_config;
5778 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5779 &video_encoder_config);
5780 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5781 vp9_settings.numberOfSpatialLayers = 3;
5782 // Since only one layer is active - automatic resize should be enabled.
5783 vp9_settings.automaticResizeOn = true;
5784 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005785 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005786 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005787 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005788 video_encoder_config.content_type =
5789 VideoEncoderConfig::ContentType::kRealtimeVideo;
5790 // Simulcast layers are used to indicate which spatial layers are active.
5791 video_encoder_config.simulcast_layers.resize(3);
5792 video_encoder_config.simulcast_layers[0].active = false;
5793 video_encoder_config.simulcast_layers[1].active = true;
5794 video_encoder_config.simulcast_layers[2].active = false;
5795
5796 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5797 kMaxPayloadLength);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005798
5799 // The encoder bitrate limits for 360p should be used.
5800 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005801 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005802 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5803 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5804 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5805 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5806 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5807 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005808 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005809 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005810 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005811 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005812
5813 // The encoder bitrate limits for 270p should be used.
5814 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02005815 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005816 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5817 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5818 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5819 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5820 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5821 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005822 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005823 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005824 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005825 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005826
5827 video_stream_encoder_->Stop();
5828}
5829
5830TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005831 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5832 VideoEncoderConfig video_encoder_config;
5833 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5834 &video_encoder_config);
5835 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5836 vp9_settings.numberOfSpatialLayers = 3;
5837 // Since only one layer is active - automatic resize should be enabled.
5838 vp9_settings.automaticResizeOn = true;
5839 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005840 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005841 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005842 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005843 video_encoder_config.content_type =
5844 VideoEncoderConfig::ContentType::kRealtimeVideo;
5845 // Simulcast layers are used to indicate which spatial layers are active.
5846 video_encoder_config.simulcast_layers.resize(3);
5847 video_encoder_config.simulcast_layers[0].active = false;
5848 video_encoder_config.simulcast_layers[1].active = true;
5849 video_encoder_config.simulcast_layers[2].active = false;
5850
5851 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5852 kMaxPayloadLength);
Åsa Persson258e9892021-02-25 10:39:51 +01005853
5854 // The default bitrate limits for 360p should be used.
5855 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005856 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5857 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005858 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005859 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005860 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5861 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5862 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5863 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5864 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5865 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005866 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005867 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005868 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005869 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005870
5871 // The default bitrate limits for 270p should be used.
5872 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005873 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5874 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005875 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
Erik Språngf449af82022-08-08 17:54:55 +02005876 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005877 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5878 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5879 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5880 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5881 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5882 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005883 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005884 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005885 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005886 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005887
5888 video_stream_encoder_->Stop();
5889}
5890
5891TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005892 webrtc::test::ScopedKeyValueConfig field_trials(
5893 field_trials_, "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
Åsa Persson258e9892021-02-25 10:39:51 +01005894 VideoEncoderConfig video_encoder_config;
5895 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5896 &video_encoder_config);
5897 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5898 vp9_settings.numberOfSpatialLayers = 3;
5899 // Since only one layer is active - automatic resize should be enabled.
5900 vp9_settings.automaticResizeOn = true;
5901 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005902 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005903 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005904 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005905 video_encoder_config.content_type =
5906 VideoEncoderConfig::ContentType::kRealtimeVideo;
5907 // Simulcast layers are used to indicate which spatial layers are active.
5908 video_encoder_config.simulcast_layers.resize(3);
5909 video_encoder_config.simulcast_layers[0].active = false;
5910 video_encoder_config.simulcast_layers[1].active = true;
5911 video_encoder_config.simulcast_layers[2].active = false;
5912
5913 // Reset encoder for field trials to take effect.
5914 ConfigureEncoder(video_encoder_config.Copy());
5915
5916 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5917 kMaxPayloadLength);
Åsa Persson258e9892021-02-25 10:39:51 +01005918
5919 // The default bitrate limits for 360p should not be used.
5920 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005921 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5922 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005923 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005924 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005925 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5926 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5927 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5928 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5929 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5930 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005931 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005932 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005933
5934 video_stream_encoder_->Stop();
5935}
5936
5937TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5938 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5939 /*num_spatial_layers=*/1, /*screenshare=*/false);
5940
5941 // The default singlecast bitrate limits for 720p should not be used.
5942 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005943 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5944 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005945 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005946 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005947 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5948 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5949 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5950 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5951 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5952 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005953 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005954 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005955
5956 video_stream_encoder_->Stop();
5957}
5958
5959TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005960 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5961 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5962 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5963 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5964 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5965 fake_encoder_.SetResolutionBitrateLimits(
5966 {kEncoderLimits180p, kEncoderLimits720p});
5967
5968 VideoEncoderConfig video_encoder_config;
5969 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5970 &video_encoder_config);
5971 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5972 vp9_settings.numberOfSpatialLayers = 3;
5973 // Since only one layer is active - automatic resize should be enabled.
5974 vp9_settings.automaticResizeOn = true;
5975 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005976 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005977 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005978 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005979 video_encoder_config.content_type =
5980 VideoEncoderConfig::ContentType::kRealtimeVideo;
5981 // Simulcast layers are used to indicate which spatial layers are active.
5982 video_encoder_config.simulcast_layers.resize(3);
5983 video_encoder_config.simulcast_layers[0].active = true;
5984 video_encoder_config.simulcast_layers[1].active = false;
5985 video_encoder_config.simulcast_layers[2].active = false;
5986
5987 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5988 kMaxPayloadLength);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005989
5990 // Limits not applied on lowest stream, limits for 180p should not be used.
5991 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
Erik Språngf449af82022-08-08 17:54:55 +02005992 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Asa Persson606d3cb2021-10-04 10:07:11 +02005993 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5994 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5995 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5996 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5997 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5998 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005999 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02006000 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01006001 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02006002 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01006003
6004 video_stream_encoder_->Stop();
6005}
6006
6007TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006008 InitialFrameDropActivatesWhenResolutionIncreases) {
6009 const int kWidth = 640;
6010 const int kHeight = 360;
6011
6012 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006013 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006014 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
6015 // Frame should not be dropped.
6016 WaitForEncodedFrame(1);
6017
6018 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006019 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006020 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
6021 // Frame should not be dropped, bitrate not too low for frame.
6022 WaitForEncodedFrame(2);
6023
6024 // Incoming resolution increases.
6025 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
6026 // Expect to drop this frame, bitrate too low for frame.
6027 ExpectDroppedFrame();
6028
6029 // Expect the sink_wants to specify a scaled frame.
6030 EXPECT_TRUE_WAIT(
6031 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
6032 video_stream_encoder_->Stop();
6033}
6034
6035TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
6036 const int kWidth = 640;
6037 const int kHeight = 360;
6038 // So that quality scaling doesn't happen by itself.
6039 fake_encoder_.SetQp(kQpHigh);
6040
6041 AdaptingFrameForwarder source(&time_controller_);
6042 source.set_adaptation_enabled(true);
6043 video_stream_encoder_->SetSource(
6044 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
6045
6046 int timestamp = 1;
6047
6048 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006049 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006050 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6051 WaitForEncodedFrame(timestamp);
6052 timestamp += 9000;
6053 // Long pause to disable all first BWE drop logic.
6054 AdvanceTime(TimeDelta::Millis(1000));
6055
6056 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006057 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006058 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6059 // Not dropped frame, as initial frame drop is disabled by now.
6060 WaitForEncodedFrame(timestamp);
6061 timestamp += 9000;
6062 AdvanceTime(TimeDelta::Millis(100));
6063
6064 // Quality adaptation down.
6065 video_stream_encoder_->TriggerQualityLow();
6066
6067 // Adaptation has an effect.
6068 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6069 5000);
6070
6071 // Frame isn't dropped as initial frame dropper is disabled.
6072 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6073 WaitForEncodedFrame(timestamp);
6074 timestamp += 9000;
6075 AdvanceTime(TimeDelta::Millis(100));
6076
6077 // Quality adaptation up.
6078 video_stream_encoder_->TriggerQualityHigh();
6079
6080 // Adaptation has an effect.
6081 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
6082 5000);
6083
6084 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6085 // Frame should not be dropped, as initial framedropper is off.
6086 WaitForEncodedFrame(timestamp);
6087
6088 video_stream_encoder_->Stop();
6089}
6090
Åsa Persson7f354f82021-02-04 15:52:15 +01006091TEST_F(VideoStreamEncoderTest,
6092 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
6093 const int kMinStartBps360p = 222000;
6094 fake_encoder_.SetResolutionBitrateLimits(
6095 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6096 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6097 800000)});
6098
6099 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6100 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6101 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6102 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
6103 0, 0, 0);
6104 // Frame should not be dropped, bitrate not too low for frame.
6105 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6106 WaitForEncodedFrame(1);
6107
6108 // Incoming resolution increases, initial frame drop activates.
6109 // Frame should be dropped, link allocation too low for frame.
6110 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6111 ExpectDroppedFrame();
6112
6113 // Expect sink_wants to specify a scaled frame.
6114 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6115 5000);
6116 video_stream_encoder_->Stop();
6117}
6118
6119TEST_F(VideoStreamEncoderTest,
6120 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6121 const int kMinStartBps360p = 222000;
6122 fake_encoder_.SetResolutionBitrateLimits(
6123 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6124 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6125 800000)});
6126
6127 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6128 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6129 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6130 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6131 0, 0, 0);
6132 // Frame should not be dropped, bitrate not too low for frame.
6133 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6134 WaitForEncodedFrame(1);
6135
6136 // Incoming resolution increases, initial frame drop activates.
6137 // Frame should be dropped, link allocation not too low for frame.
6138 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6139 WaitForEncodedFrame(2);
6140
6141 video_stream_encoder_->Stop();
6142}
6143
Åsa Perssone644a032019-11-08 15:56:00 +01006144TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006145 webrtc::test::ScopedKeyValueConfig field_trials(
6146 field_trials_,
Åsa Persson06defc42021-09-10 15:28:48 +02006147 "WebRTC-Video-QualityRampupSettings/"
6148 "min_pixels:921600,min_duration_ms:2000/");
6149
6150 const int kWidth = 1280;
6151 const int kHeight = 720;
6152 const int kFps = 10;
6153 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006154
6155 // Reset encoder for field trials to take effect.
6156 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 10:07:11 +02006157 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 11:34:32 +02006158 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01006159 ConfigureEncoder(std::move(config));
6160 fake_encoder_.SetQp(kQpLow);
6161
6162 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006163 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01006164 source.set_adaptation_enabled(true);
6165 video_stream_encoder_->SetSource(&source,
6166 DegradationPreference::MAINTAIN_FRAMERATE);
6167
6168 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 10:07:11 +02006169 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 18:49:07 +02006170 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006171 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006172
6173 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 15:28:48 +02006174 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006175 int64_t timestamp_ms = kFrameIntervalMs;
6176 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6177 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02006178 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6179 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01006180
6181 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02006182 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6183 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006184
Artem Titovab30d722021-07-27 16:22:11 +02006185 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 15:28:48 +02006186 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 15:56:00 +01006187 for (size_t i = 1; i <= 10; i++) {
6188 timestamp_ms += kFrameIntervalMs;
6189 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6190 WaitForEncodedFrame(timestamp_ms);
6191 }
Åsa Persson06defc42021-09-10 15:28:48 +02006192
6193 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6194 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6195 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6196 timestamp_ms += kFrameIntervalMs;
6197 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6198 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 15:56:00 +01006199 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6200 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6201
Åsa Persson06defc42021-09-10 15:28:48 +02006202 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 15:56:00 +01006203 timestamp_ms += kFrameIntervalMs;
6204 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6205 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006206 // The ramp-up code involves the adaptation queue, give it time to execute.
6207 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006208 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006209 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006210
6211 // Frame should not be adapted.
6212 timestamp_ms += kFrameIntervalMs;
6213 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6214 WaitForEncodedFrame(kWidth, kHeight);
6215 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6216
6217 video_stream_encoder_->Stop();
6218}
6219
mflodmancc3d4422017-08-03 08:27:51 -07006220TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006221 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006222 webrtc::test::ScopedKeyValueConfig field_trials(
6223 field_trials_, "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006224 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006225 source.set_adaptation_enabled(true);
6226 video_stream_encoder_->SetSource(&source,
6227 DegradationPreference::MAINTAIN_FRAMERATE);
6228 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006229 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006230 fake_encoder_.SetQp(kQpHigh + 1);
6231 const int kWidth = 1280;
6232 const int kHeight = 720;
6233 const int64_t kFrameIntervalMs = 100;
6234 int64_t timestamp_ms = kFrameIntervalMs;
6235 for (size_t i = 1; i <= 100; i++) {
6236 timestamp_ms += kFrameIntervalMs;
6237 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6238 WaitForEncodedFrame(timestamp_ms);
6239 }
6240 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6241 // for the first time.
6242 // TODO(eshr): We should avoid these waits by using threads with simulated
6243 // time.
6244 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6245 2000 * 2.5 * 2);
6246 timestamp_ms += kFrameIntervalMs;
6247 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6248 WaitForEncodedFrame(timestamp_ms);
6249 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6250 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6251 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6252
6253 // Disable Quality scaling by turning off scaler on the encoder and
6254 // reconfiguring.
6255 fake_encoder_.SetQualityScaling(false);
6256 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6257 kMaxPayloadLength);
6258 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Markus Handell28c71802021-11-08 10:11:55 +01006259 AdvanceTime(TimeDelta::Zero());
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006260 // Since we turned off the quality scaler, the adaptations made by it are
6261 // removed.
6262 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6263 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6264
6265 video_stream_encoder_->Stop();
6266}
6267
6268TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006269 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6270 const int kTooSmallWidth = 10;
6271 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006272 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006273 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006274
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006275 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006276 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006277 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006278 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006279 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006280 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6281
6282 // Trigger adapt down, too small frame, expect no change.
6283 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006284 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006285 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006286 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006287 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6288 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6289
mflodmancc3d4422017-08-03 08:27:51 -07006290 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006291}
6292
mflodmancc3d4422017-08-03 08:27:51 -07006293TEST_F(VideoStreamEncoderTest,
6294 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006295 const int kTooSmallWidth = 10;
6296 const int kTooSmallHeight = 10;
6297 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006298 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006299 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006300
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006301 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006302 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006303 video_stream_encoder_->SetSource(&source,
6304 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006305 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006306 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6307 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6308
6309 // Trigger adapt down, expect limited framerate.
6310 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006311 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006312 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006313 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006314 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6315 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6316 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6317
6318 // Trigger adapt down, too small frame, expect no change.
6319 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006320 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006321 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006322 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006323 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6324 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6325 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6326
mflodmancc3d4422017-08-03 08:27:51 -07006327 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006328}
6329
mflodmancc3d4422017-08-03 08:27:51 -07006330TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006331 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006332 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006333 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006334 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006335 const int kFrameWidth = 1280;
6336 const int kFrameHeight = 720;
6337 video_source_.IncomingCapturedFrame(
6338 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006339 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006340 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006341}
6342
sprangb1ca0732017-02-01 08:38:12 -08006343// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006344TEST_F(VideoStreamEncoderTest,
6345 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006346 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006347 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006348
6349 const int kFrameWidth = 1280;
6350 const int kFrameHeight = 720;
6351 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006352 // requested by
6353 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006354 video_source_.set_adaptation_enabled(true);
6355
6356 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006357 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006358 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006359
6360 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006361 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006362 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006363 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006364 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006365
asaperssonfab67072017-04-04 05:51:49 -07006366 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006367 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006368 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006369 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006370 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006371
mflodmancc3d4422017-08-03 08:27:51 -07006372 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006373}
sprangfe627f32017-03-29 08:24:59 -07006374
mflodmancc3d4422017-08-03 08:27:51 -07006375TEST_F(VideoStreamEncoderTest,
6376 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006377 const int kFrameWidth = 1280;
6378 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006379
Henrik Boström381d1092020-05-12 18:49:07 +02006380 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006381 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006382 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006383 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006384 video_source_.set_adaptation_enabled(true);
6385
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006386 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006387
6388 video_source_.IncomingCapturedFrame(
6389 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006390 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006391
6392 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006393 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006394
6395 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006396 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006397 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006398 video_source_.IncomingCapturedFrame(
6399 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006400 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006401 }
6402
6403 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006404 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006405 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006406 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006407 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006408 video_source_.IncomingCapturedFrame(
6409 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Markus Handell2cfc1af2022-08-19 08:16:48 +00006410 if (!WaitForFrame(kFrameTimeout)) {
sprangc5d62e22017-04-02 23:53:04 -07006411 ++num_frames_dropped;
6412 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006413 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006414 }
6415 }
6416
sprang4847ae62017-06-27 07:06:52 -07006417 // Add some slack to account for frames dropped by the frame dropper.
6418 const int kErrorMargin = 1;
6419 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006420 kErrorMargin);
6421
6422 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006423 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006424 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006425 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006426 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006427 video_source_.IncomingCapturedFrame(
6428 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Markus Handell2cfc1af2022-08-19 08:16:48 +00006429 if (!WaitForFrame(kFrameTimeout)) {
sprangc5d62e22017-04-02 23:53:04 -07006430 ++num_frames_dropped;
6431 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006432 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006433 }
6434 }
sprang4847ae62017-06-27 07:06:52 -07006435 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006436 kErrorMargin);
6437
6438 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006439 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006440 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006441 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006442 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006443 video_source_.IncomingCapturedFrame(
6444 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Markus Handell2cfc1af2022-08-19 08:16:48 +00006445 if (!WaitForFrame(kFrameTimeout)) {
sprangc5d62e22017-04-02 23:53:04 -07006446 ++num_frames_dropped;
6447 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006448 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006449 }
6450 }
sprang4847ae62017-06-27 07:06:52 -07006451 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006452 kErrorMargin);
6453
6454 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006455 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006456 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006457 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006458 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006459 video_source_.IncomingCapturedFrame(
6460 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Markus Handell2cfc1af2022-08-19 08:16:48 +00006461 if (!WaitForFrame(kFrameTimeout)) {
sprangc5d62e22017-04-02 23:53:04 -07006462 ++num_frames_dropped;
6463 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006464 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006465 }
6466 }
6467 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6468
mflodmancc3d4422017-08-03 08:27:51 -07006469 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006470}
6471
mflodmancc3d4422017-08-03 08:27:51 -07006472TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006473 const int kFramerateFps = 5;
6474 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006475 const int kFrameWidth = 1280;
6476 const int kFrameHeight = 720;
6477
sprang4847ae62017-06-27 07:06:52 -07006478 // Reconfigure encoder with two temporal layers and screensharing, which will
6479 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006480 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006481
Henrik Boström381d1092020-05-12 18:49:07 +02006482 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006483 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006484 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006485 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006486 video_source_.set_adaptation_enabled(true);
6487
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006488 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006489
6490 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006491 rtc::VideoSinkWants last_wants;
6492 do {
6493 last_wants = video_source_.sink_wants();
6494
sprangc5d62e22017-04-02 23:53:04 -07006495 // Insert frames to get a new fps estimate...
6496 for (int j = 0; j < kFramerateFps; ++j) {
6497 video_source_.IncomingCapturedFrame(
6498 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006499 if (video_source_.last_sent_width()) {
6500 sink_.WaitForEncodedFrame(timestamp_ms);
6501 }
sprangc5d62e22017-04-02 23:53:04 -07006502 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006503 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006504 }
6505 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006506 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006507 } while (video_source_.sink_wants().max_framerate_fps <
6508 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006509
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006510 EXPECT_THAT(video_source_.sink_wants(),
6511 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006512
mflodmancc3d4422017-08-03 08:27:51 -07006513 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006514}
asaperssonf7e294d2017-06-13 23:25:22 -07006515
mflodmancc3d4422017-08-03 08:27:51 -07006516TEST_F(VideoStreamEncoderTest,
6517 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006518 const int kWidth = 1280;
6519 const int kHeight = 720;
6520 const int64_t kFrameIntervalMs = 150;
6521 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006522 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006523 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006524
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006525 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006526 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006527 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006528 video_stream_encoder_->SetSource(&source,
6529 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006530 timestamp_ms += kFrameIntervalMs;
6531 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006532 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006533 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006534 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6535 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6536 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6537
6538 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006539 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006540 timestamp_ms += kFrameIntervalMs;
6541 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006542 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006543 EXPECT_THAT(source.sink_wants(),
6544 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006545 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6546 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6547 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6548
6549 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006550 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006551 timestamp_ms += kFrameIntervalMs;
6552 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006553 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006554 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006555 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6556 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6557 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6558
6559 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006560 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006561 timestamp_ms += kFrameIntervalMs;
6562 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006563 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006564 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006565 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6566 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6567 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6568
6569 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006570 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006571 timestamp_ms += kFrameIntervalMs;
6572 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006573 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006574 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006575 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6576 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6577 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6578
6579 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006580 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006581 timestamp_ms += kFrameIntervalMs;
6582 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006583 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006584 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006585 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6586 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6587 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6588
6589 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006590 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006591 timestamp_ms += kFrameIntervalMs;
6592 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006593 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006594 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006595 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6596 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6597 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6598
6599 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006600 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006601 timestamp_ms += kFrameIntervalMs;
6602 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006603 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006604 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006605 rtc::VideoSinkWants last_wants = source.sink_wants();
6606 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6607 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6608 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6609
6610 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006611 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006612 timestamp_ms += kFrameIntervalMs;
6613 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006614 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006615 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006616 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6617 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6618 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6619
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006620 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006621 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006622 timestamp_ms += kFrameIntervalMs;
6623 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006624 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006625 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006626 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6627 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6628 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6629
6630 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006631 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006632 timestamp_ms += kFrameIntervalMs;
6633 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006634 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006635 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006636 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6637 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6638 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6639
6640 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006641 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006642 timestamp_ms += kFrameIntervalMs;
6643 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006644 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006645 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006646 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6647 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6648 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6649
6650 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006651 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006652 timestamp_ms += kFrameIntervalMs;
6653 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006654 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006655 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006656 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6657 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6658 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6659
6660 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006661 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006662 timestamp_ms += kFrameIntervalMs;
6663 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006664 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006665 EXPECT_THAT(source.sink_wants(), FpsMax());
6666 EXPECT_EQ(source.sink_wants().max_pixel_count,
6667 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006668 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6669 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6670 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6671
6672 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006673 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006674 timestamp_ms += kFrameIntervalMs;
6675 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006676 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006677 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006678 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6679 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6680 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6681
Åsa Persson30ab0152019-08-27 12:22:33 +02006682 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006683 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006684 timestamp_ms += kFrameIntervalMs;
6685 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006686 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006687 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006688 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006689 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6690 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6691 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6692
6693 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006694 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006695 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006696 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6697
mflodmancc3d4422017-08-03 08:27:51 -07006698 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006699}
6700
mflodmancc3d4422017-08-03 08:27:51 -07006701TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006702 const int kWidth = 1280;
6703 const int kHeight = 720;
6704 const int64_t kFrameIntervalMs = 150;
6705 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006706 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006707 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006708
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006709 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006710 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006711 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006712 video_stream_encoder_->SetSource(&source,
6713 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006714 timestamp_ms += kFrameIntervalMs;
6715 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006716 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006717 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006718 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6719 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6720 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6721 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6722 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6723 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6724
6725 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006726 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006727 timestamp_ms += kFrameIntervalMs;
6728 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006729 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006730 EXPECT_THAT(source.sink_wants(),
6731 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006732 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6733 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6734 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6735 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6736 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6737 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6738
6739 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006740 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006741 timestamp_ms += kFrameIntervalMs;
6742 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006743 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006744 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006745 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6746 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6747 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6748 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6749 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6750 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6751
6752 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006753 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006754 timestamp_ms += kFrameIntervalMs;
6755 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006756 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006757 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006758 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006759 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6760 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6761 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6762 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6763 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6764
Evan Shrubsole64469032020-06-11 10:45:29 +02006765 // Trigger cpu adapt up, expect no change since QP is most limited.
6766 {
6767 // Store current sink wants since we expect no change and if there is no
6768 // change then last_wants() is not updated.
6769 auto previous_sink_wants = source.sink_wants();
6770 video_stream_encoder_->TriggerCpuUnderuse();
6771 timestamp_ms += kFrameIntervalMs;
6772 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6773 WaitForEncodedFrame(timestamp_ms);
6774 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6775 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6776 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6777 }
6778
6779 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6780 video_stream_encoder_->TriggerQualityHigh();
6781 timestamp_ms += kFrameIntervalMs;
6782 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6783 WaitForEncodedFrame(timestamp_ms);
6784 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6785 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6786 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6787 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6788 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6789 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6790 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6791
6792 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6793 // expect increased resolution (960x540@30fps).
6794 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006795 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006796 timestamp_ms += kFrameIntervalMs;
6797 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006798 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006799 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006800 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6801 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6802 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6803 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6804 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006805 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006806
Evan Shrubsole64469032020-06-11 10:45:29 +02006807 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6808 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006809 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006810 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006811 timestamp_ms += kFrameIntervalMs;
6812 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006813 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006814 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006815 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006816 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6817 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6818 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6819 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6820 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006821 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006822
6823 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006824 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006825 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006826 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006827 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006828
mflodmancc3d4422017-08-03 08:27:51 -07006829 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006830}
6831
mflodmancc3d4422017-08-03 08:27:51 -07006832TEST_F(VideoStreamEncoderTest,
6833 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006834 const int kWidth = 640;
6835 const int kHeight = 360;
6836 const int kFpsLimit = 15;
6837 const int64_t kFrameIntervalMs = 150;
6838 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006839 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006840 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006841
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006842 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006843 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006844 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006845 video_stream_encoder_->SetSource(&source,
6846 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006847 timestamp_ms += kFrameIntervalMs;
6848 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006849 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006850 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006851 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6852 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6853 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6854 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6855 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6856 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6857
6858 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006859 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006860 timestamp_ms += kFrameIntervalMs;
6861 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006862 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006863 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006864 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6865 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6866 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6867 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6868 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6869 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6870
6871 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006872 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006873 timestamp_ms += kFrameIntervalMs;
6874 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006875 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006876 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006877 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006878 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006879 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6880 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6881 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6882 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6883
Evan Shrubsole64469032020-06-11 10:45:29 +02006884 // Trigger cpu adapt up, expect no change because quality is most limited.
6885 {
6886 auto previous_sink_wants = source.sink_wants();
6887 // Store current sink wants since we expect no change ind if there is no
6888 // change then last__wants() is not updated.
6889 video_stream_encoder_->TriggerCpuUnderuse();
6890 timestamp_ms += kFrameIntervalMs;
6891 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6892 WaitForEncodedFrame(timestamp_ms);
6893 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6894 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6895 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6896 }
6897
6898 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6899 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006900 timestamp_ms += kFrameIntervalMs;
6901 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006902 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006903 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006904 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6905 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6906 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006907 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6908 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6909 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006910
Evan Shrubsole64469032020-06-11 10:45:29 +02006911 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006912 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006913 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006914 timestamp_ms += kFrameIntervalMs;
6915 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006916 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006917 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006918 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6919 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6920 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6921 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6922 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006923 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006924
6925 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006926 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006927 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006928 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006929 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006930
mflodmancc3d4422017-08-03 08:27:51 -07006931 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006932}
6933
mflodmancc3d4422017-08-03 08:27:51 -07006934TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006935 const int kFrameWidth = 1920;
6936 const int kFrameHeight = 1080;
Ying Wangdb0d5862022-04-28 19:26:22 +02006937 // 2/3 of 1920.
6938 const int kAdaptedFrameWidth = 1280;
6939 // 2/3 of 1080.
6940 const int kAdaptedFrameHeight = 720;
ilnik6b826ef2017-06-16 06:53:48 -07006941 const int kFramerate = 24;
6942
Henrik Boström381d1092020-05-12 18:49:07 +02006943 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006944 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006945 // Trigger reconfigure encoder (without resetting the entire instance).
6946 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006947 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6948 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02006949 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 06:53:48 -07006950 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006951 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006952 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006953 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006954 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006955
6956 video_source_.set_adaptation_enabled(true);
6957
6958 video_source_.IncomingCapturedFrame(
6959 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006960 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006961
6962 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006963 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006964 video_source_.IncomingCapturedFrame(
6965 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006966 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006967
mflodmancc3d4422017-08-03 08:27:51 -07006968 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006969}
6970
mflodmancc3d4422017-08-03 08:27:51 -07006971TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006972 const int kFrameWidth = 1280;
6973 const int kFrameHeight = 720;
6974 const int kLowFps = 2;
6975 const int kHighFps = 30;
6976
Henrik Boström381d1092020-05-12 18:49:07 +02006977 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006978 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006979
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006980 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006981 max_framerate_ = kLowFps;
6982
6983 // Insert 2 seconds of 2fps video.
6984 for (int i = 0; i < kLowFps * 2; ++i) {
6985 video_source_.IncomingCapturedFrame(
6986 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6987 WaitForEncodedFrame(timestamp_ms);
6988 timestamp_ms += 1000 / kLowFps;
6989 }
6990
6991 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006992 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006993 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006994 video_source_.IncomingCapturedFrame(
6995 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6996 WaitForEncodedFrame(timestamp_ms);
6997 timestamp_ms += 1000 / kLowFps;
6998
6999 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
7000
7001 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02007002 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
Markus Handell2cfc1af2022-08-19 08:16:48 +00007003 constexpr TimeDelta kFrameInterval = TimeDelta::Seconds(1) / kHighFps;
sprang4847ae62017-06-27 07:06:52 -07007004 max_framerate_ = kHighFps;
7005 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
7006 video_source_.IncomingCapturedFrame(
7007 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7008 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
7009 // be dropped if the encoder hans't been updated with the new higher target
7010 // framerate yet, causing it to overshoot the target bitrate and then
7011 // suffering the wrath of the media optimizer.
Markus Handell2cfc1af2022-08-19 08:16:48 +00007012 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameInterval);
7013 timestamp_ms += kFrameInterval.ms();
sprang4847ae62017-06-27 07:06:52 -07007014 }
7015
7016 // Don expect correct measurement just yet, but it should be higher than
7017 // before.
7018 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
7019
mflodmancc3d4422017-08-03 08:27:51 -07007020 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007021}
7022
mflodmancc3d4422017-08-03 08:27:51 -07007023TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07007024 const int kFrameWidth = 1280;
7025 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 15:09:05 +02007026 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01007027 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02007028 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07007029
Henrik Boström381d1092020-05-12 18:49:07 +02007030 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007031 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07007032 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07007033
7034 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007035 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07007036 video_source_.IncomingCapturedFrame(
7037 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7038 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02007039 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007040
7041 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02007042 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007043 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07007044
7045 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02007046 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007047 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07007048
Per Kjellanderdcef6412020-10-07 15:09:05 +02007049 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07007050 video_source_.IncomingCapturedFrame(
7051 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7052 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02007053 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007054
mflodmancc3d4422017-08-03 08:27:51 -07007055 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007056}
ilnik6b826ef2017-06-16 06:53:48 -07007057
Niels Möller4db138e2018-04-19 09:04:13 +02007058TEST_F(VideoStreamEncoderTest,
7059 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
7060 const int kFrameWidth = 1280;
7061 const int kFrameHeight = 720;
Markus Handell8e4197b2022-05-30 15:45:28 +02007062 const test::ScopedKeyValueConfig kFieldTrials;
7063 const CpuOveruseOptions default_options(kFieldTrials);
Henrik Boström381d1092020-05-12 18:49:07 +02007064 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007065 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007066 video_source_.IncomingCapturedFrame(
7067 CreateFrame(1, kFrameWidth, kFrameHeight));
7068 WaitForEncodedFrame(1);
7069 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7070 .low_encode_usage_threshold_percent,
7071 default_options.low_encode_usage_threshold_percent);
7072 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7073 .high_encode_usage_threshold_percent,
7074 default_options.high_encode_usage_threshold_percent);
7075 video_stream_encoder_->Stop();
7076}
7077
7078TEST_F(VideoStreamEncoderTest,
7079 HigherCpuAdaptationThresholdsForHardwareEncoder) {
7080 const int kFrameWidth = 1280;
7081 const int kFrameHeight = 720;
Markus Handell8e4197b2022-05-30 15:45:28 +02007082 const test::ScopedKeyValueConfig kFieldTrials;
7083 CpuOveruseOptions hardware_options(kFieldTrials);
Niels Möller4db138e2018-04-19 09:04:13 +02007084 hardware_options.low_encode_usage_threshold_percent = 150;
7085 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01007086 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02007087
Henrik Boström381d1092020-05-12 18:49:07 +02007088 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007089 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007090 video_source_.IncomingCapturedFrame(
7091 CreateFrame(1, kFrameWidth, kFrameHeight));
7092 WaitForEncodedFrame(1);
7093 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7094 .low_encode_usage_threshold_percent,
7095 hardware_options.low_encode_usage_threshold_percent);
7096 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7097 .high_encode_usage_threshold_percent,
7098 hardware_options.high_encode_usage_threshold_percent);
7099 video_stream_encoder_->Stop();
7100}
7101
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007102TEST_F(VideoStreamEncoderTest,
7103 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
7104 const int kFrameWidth = 1280;
7105 const int kFrameHeight = 720;
7106
Markus Handell8e4197b2022-05-30 15:45:28 +02007107 const test::ScopedKeyValueConfig kFieldTrials;
7108 const CpuOveruseOptions default_options(kFieldTrials);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007109 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007110 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007111 video_source_.IncomingCapturedFrame(
7112 CreateFrame(1, kFrameWidth, kFrameHeight));
7113 WaitForEncodedFrame(1);
7114 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7115 .low_encode_usage_threshold_percent,
7116 default_options.low_encode_usage_threshold_percent);
7117 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7118 .high_encode_usage_threshold_percent,
7119 default_options.high_encode_usage_threshold_percent);
7120
Markus Handell8e4197b2022-05-30 15:45:28 +02007121 CpuOveruseOptions hardware_options(kFieldTrials);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007122 hardware_options.low_encode_usage_threshold_percent = 150;
7123 hardware_options.high_encode_usage_threshold_percent = 200;
7124 fake_encoder_.SetIsHardwareAccelerated(true);
7125
7126 video_source_.IncomingCapturedFrame(
7127 CreateFrame(2, kFrameWidth, kFrameHeight));
7128 WaitForEncodedFrame(2);
7129
7130 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7131 .low_encode_usage_threshold_percent,
7132 hardware_options.low_encode_usage_threshold_percent);
7133 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7134 .high_encode_usage_threshold_percent,
7135 hardware_options.high_encode_usage_threshold_percent);
7136
7137 video_stream_encoder_->Stop();
7138}
7139
Niels Möller6bb5ab92019-01-11 11:11:10 +01007140TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7141 const int kFrameWidth = 320;
7142 const int kFrameHeight = 240;
7143 const int kFps = 30;
Asa Persson606d3cb2021-10-04 10:07:11 +02007144 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007145 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7146
Henrik Boström381d1092020-05-12 18:49:07 +02007147 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007148 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007149
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007150 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007151 max_framerate_ = kFps;
7152
7153 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7154 fake_encoder_.SimulateOvershoot(1.0);
7155 int num_dropped = 0;
7156 for (int i = 0; i < kNumFramesInRun; ++i) {
7157 video_source_.IncomingCapturedFrame(
7158 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7159 // Wait up to two frame durations for a frame to arrive.
Markus Handell2cfc1af2022-08-19 08:16:48 +00007160 if (!TimedWaitForEncodedFrame(timestamp_ms,
7161 2 * TimeDelta::Seconds(1) / kFps)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01007162 ++num_dropped;
7163 }
7164 timestamp_ms += 1000 / kFps;
7165 }
7166
Erik Språnga8d48ab2019-02-08 14:17:40 +01007167 // Framerate should be measured to be near the expected target rate.
7168 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7169
7170 // Frame drops should be within 5% of expected 0%.
7171 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007172
7173 // Make encoder produce frames at double the expected bitrate during 3 seconds
7174 // of video, verify number of drops. Rate needs to be slightly changed in
7175 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01007176 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02007177 const RateControlSettings trials =
7178 RateControlSettings::ParseFromFieldTrials();
7179 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01007180 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02007181 // frame dropping since the adjuter will try to just lower the target
7182 // bitrate rather than drop frames. If network headroom can be used, it
7183 // doesn't push back as hard so we don't need quite as much overshoot.
7184 // These numbers are unfortunately a bit magical but there's not trivial
7185 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01007186 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01007187 }
7188 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02007189 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007190 kTargetBitrate + DataRate::KilobitsPerSec(1),
7191 kTargetBitrate + DataRate::KilobitsPerSec(1),
7192 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007193 num_dropped = 0;
7194 for (int i = 0; i < kNumFramesInRun; ++i) {
7195 video_source_.IncomingCapturedFrame(
7196 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7197 // Wait up to two frame durations for a frame to arrive.
Markus Handell2cfc1af2022-08-19 08:16:48 +00007198 if (!TimedWaitForEncodedFrame(timestamp_ms,
7199 2 * TimeDelta::Seconds(1) / kFps)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01007200 ++num_dropped;
7201 }
7202 timestamp_ms += 1000 / kFps;
7203 }
7204
Henrik Boström381d1092020-05-12 18:49:07 +02007205 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007206 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007207
7208 // Target framerate should be still be near the expected target, despite
7209 // the frame drops.
7210 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7211
7212 // Frame drops should be within 5% of expected 50%.
7213 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007214
7215 video_stream_encoder_->Stop();
7216}
7217
7218TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7219 const int kFrameWidth = 320;
7220 const int kFrameHeight = 240;
7221 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 10:07:11 +02007222 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007223
7224 ASSERT_GT(max_framerate_, kActualInputFps);
7225
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007226 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007227 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007228 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007229 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007230
7231 // Insert 3 seconds of video, with an input fps lower than configured max.
7232 for (int i = 0; i < kActualInputFps * 3; ++i) {
7233 video_source_.IncomingCapturedFrame(
7234 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7235 // Wait up to two frame durations for a frame to arrive.
7236 WaitForEncodedFrame(timestamp_ms);
7237 timestamp_ms += 1000 / kActualInputFps;
7238 }
7239
7240 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7241
7242 video_stream_encoder_->Stop();
7243}
7244
Markus Handell9a478b52021-11-18 16:07:01 +01007245TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007246 VideoFrame::UpdateRect rect;
Markus Handell9a478b52021-11-18 16:07:01 +01007247 test::FrameForwarder source;
7248 video_stream_encoder_->SetSource(&source,
7249 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02007250 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007251 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007252
Markus Handell9a478b52021-11-18 16:07:01 +01007253 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(1, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007254 WaitForEncodedFrame(1);
7255 // On the very first frame full update should be forced.
7256 rect = fake_encoder_.GetLastUpdateRect();
7257 EXPECT_EQ(rect.offset_x, 0);
7258 EXPECT_EQ(rect.offset_y, 0);
7259 EXPECT_EQ(rect.height, codec_height_);
7260 EXPECT_EQ(rect.width, codec_width_);
Markus Handell9a478b52021-11-18 16:07:01 +01007261 // Frame with NTP timestamp 2 will be dropped due to outstanding frames
7262 // scheduled for processing during encoder queue processing of frame 2.
7263 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(2, nullptr, 1));
7264 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(3, nullptr, 10));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007265 WaitForEncodedFrame(3);
7266 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7267 rect = fake_encoder_.GetLastUpdateRect();
7268 EXPECT_EQ(rect.offset_x, 1);
7269 EXPECT_EQ(rect.offset_y, 0);
7270 EXPECT_EQ(rect.width, 10);
7271 EXPECT_EQ(rect.height, 1);
7272
Markus Handell9a478b52021-11-18 16:07:01 +01007273 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(4, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007274 WaitForEncodedFrame(4);
7275 // Previous frame was encoded, so no accumulation should happen.
7276 rect = fake_encoder_.GetLastUpdateRect();
7277 EXPECT_EQ(rect.offset_x, 0);
7278 EXPECT_EQ(rect.offset_y, 0);
7279 EXPECT_EQ(rect.width, 1);
7280 EXPECT_EQ(rect.height, 1);
7281
7282 video_stream_encoder_->Stop();
7283}
7284
Erik Språngd7329ca2019-02-21 21:19:53 +01007285TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007286 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007287 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007288
7289 // First frame is always keyframe.
7290 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7291 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007292 EXPECT_THAT(
7293 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007294 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007295
7296 // Insert delta frame.
7297 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7298 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007299 EXPECT_THAT(
7300 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007301 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007302
7303 // Request next frame be a key-frame.
7304 video_stream_encoder_->SendKeyFrame();
7305 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7306 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007307 EXPECT_THAT(
7308 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007309 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007310
7311 video_stream_encoder_->Stop();
7312}
7313
7314TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7315 // Setup simulcast with three streams.
7316 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007317 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007318 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7319 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007320 // Wait for all three layers before triggering event.
7321 sink_.SetNumExpectedLayers(3);
7322
7323 // First frame is always keyframe.
7324 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7325 WaitForEncodedFrame(1);
7326 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007327 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7328 VideoFrameType::kVideoFrameKey,
7329 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007330
7331 // Insert delta frame.
7332 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7333 WaitForEncodedFrame(2);
7334 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007335 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7336 VideoFrameType::kVideoFrameDelta,
7337 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007338
7339 // Request next frame be a key-frame.
7340 // Only first stream is configured to produce key-frame.
7341 video_stream_encoder_->SendKeyFrame();
7342 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7343 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007344
7345 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7346 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007347 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007348 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007349 VideoFrameType::kVideoFrameKey,
7350 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007351
7352 video_stream_encoder_->Stop();
7353}
7354
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007355TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007356 // SPS contains VUI with restrictions on the maximum number of reordered
7357 // pictures, there is no need to rewrite the bitstream to enable faster
7358 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007359 ResetEncoder("H264", 1, 1, 1, false);
7360
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007361 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007362 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007363 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007364
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007365 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 10:07:11 +02007366 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007367
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007368 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7369 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007370
7371 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007372 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007373
7374 video_stream_encoder_->Stop();
7375}
7376
7377TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007378 // SPS does not contain VUI, the bitstream is will be rewritten with added
7379 // VUI with restrictions on the maximum number of reordered pictures to
7380 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007381 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7382 0x00, 0x00, 0x03, 0x03, 0xF4,
7383 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007384 ResetEncoder("H264", 1, 1, 1, false);
7385
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007386 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007387 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007388 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007389
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007390 fake_encoder_.SetEncodedImageData(
7391 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007392
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007393 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7394 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007395
7396 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007397 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007398
7399 video_stream_encoder_->Stop();
7400}
7401
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007402TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7403 const int kFrameWidth = 1280;
7404 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 10:07:11 +02007405 const DataRate kTargetBitrate =
7406 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007407
Henrik Boström381d1092020-05-12 18:49:07 +02007408 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007409 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007410 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7411
7412 // Insert a first video frame. It should be dropped because of downscale in
7413 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007414 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007415 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7416 frame.set_rotation(kVideoRotation_270);
7417 video_source_.IncomingCapturedFrame(frame);
7418
7419 ExpectDroppedFrame();
7420
7421 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007422 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007423 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7424 frame.set_rotation(kVideoRotation_90);
7425 video_source_.IncomingCapturedFrame(frame);
7426
7427 WaitForEncodedFrame(timestamp_ms);
7428 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7429
7430 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007431 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007432 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7433 frame.set_rotation(kVideoRotation_180);
7434 video_source_.IncomingCapturedFrame(frame);
7435
7436 WaitForEncodedFrame(timestamp_ms);
7437 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7438
7439 video_stream_encoder_->Stop();
7440}
7441
Erik Språng5056af02019-09-02 15:53:11 +02007442TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7443 const int kFrameWidth = 320;
7444 const int kFrameHeight = 180;
7445
7446 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007447 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007448 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7449 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7450 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007451 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007452 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007453 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007454
7455 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007456 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007457 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7458 frame.set_rotation(kVideoRotation_270);
7459 video_source_.IncomingCapturedFrame(frame);
7460 WaitForEncodedFrame(timestamp_ms);
7461
7462 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 10:07:11 +02007463 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007464 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7465 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007466 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007467 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007468 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007469 /*link_allocation=*/target_rate,
7470 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007471 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007472 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007473 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7474
7475 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7476 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7477 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007478 DataRate allocation_sum =
7479 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007480 EXPECT_EQ(min_rate, allocation_sum);
7481 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7482
7483 video_stream_encoder_->Stop();
7484}
7485
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007486TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007487 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007488 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007489 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007490 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007491 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7492 WaitForEncodedFrame(1);
7493
7494 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7495 ASSERT_TRUE(prev_rate_settings.has_value());
7496 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7497 kDefaultFramerate);
7498
7499 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7500 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7501 timestamp_ms += 1000 / kDefaultFramerate;
7502 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7503 WaitForEncodedFrame(timestamp_ms);
7504 }
7505 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7506 kDefaultFramerate);
7507 // Capture larger frame to trigger a reconfigure.
7508 codec_height_ *= 2;
7509 codec_width_ *= 2;
7510 timestamp_ms += 1000 / kDefaultFramerate;
7511 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7512 WaitForEncodedFrame(timestamp_ms);
7513
7514 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7515 auto current_rate_settings =
7516 fake_encoder_.GetAndResetLastRateControlSettings();
7517 // Ensure we have actually reconfigured twice
7518 // The rate settings should have been set again even though
7519 // they haven't changed.
7520 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007521 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007522
7523 video_stream_encoder_->Stop();
7524}
7525
philipeld9cc8c02019-09-16 14:53:40 +02007526struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007527 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007528 MOCK_METHOD(void,
7529 RequestEncoderSwitch,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007530 (const webrtc::SdpVideoFormat& format,
7531 bool allow_default_fallback),
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007532 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007533};
7534
philipel9b058032020-02-10 11:30:00 +01007535TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7536 constexpr int kDontCare = 100;
7537 StrictMock<MockEncoderSelector> encoder_selector;
7538 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7539 &fake_encoder_, &encoder_selector);
7540 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7541
7542 // Reset encoder for new configuration to take effect.
7543 ConfigureEncoder(video_encoder_config_.Copy());
7544
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007545 EXPECT_CALL(encoder_selector, OnCurrentEncoder);
philipel9b058032020-02-10 11:30:00 +01007546
7547 video_source_.IncomingCapturedFrame(
7548 CreateFrame(kDontCare, kDontCare, kDontCare));
Markus Handell28c71802021-11-08 10:11:55 +01007549 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007550 video_stream_encoder_->Stop();
7551
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007552 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007553 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007554 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7555 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007556 video_stream_encoder_.reset();
7557}
7558
7559TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7560 constexpr int kDontCare = 100;
7561
7562 NiceMock<MockEncoderSelector> encoder_selector;
7563 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7564 video_send_config_.encoder_settings.encoder_switch_request_callback =
7565 &switch_callback;
7566 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7567 &fake_encoder_, &encoder_selector);
7568 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7569
7570 // Reset encoder for new configuration to take effect.
7571 ConfigureEncoder(video_encoder_config_.Copy());
7572
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007573 ON_CALL(encoder_selector, OnAvailableBitrate)
philipel9b058032020-02-10 11:30:00 +01007574 .WillByDefault(Return(SdpVideoFormat("AV1")));
7575 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007576 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7577 /*allow_default_fallback=*/false));
philipel9b058032020-02-10 11:30:00 +01007578
Henrik Boström381d1092020-05-12 18:49:07 +02007579 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007580 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7581 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7582 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007583 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007584 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007585 /*cwnd_reduce_ratio=*/0);
Markus Handell28c71802021-11-08 10:11:55 +01007586 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007587
7588 video_stream_encoder_->Stop();
7589}
7590
philipel6daa3042022-04-11 10:48:28 +02007591TEST_F(VideoStreamEncoderTest, EncoderSelectorResolutionSwitch) {
7592 NiceMock<MockEncoderSelector> encoder_selector;
7593 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7594 video_send_config_.encoder_settings.encoder_switch_request_callback =
7595 &switch_callback;
7596 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7597 &fake_encoder_, &encoder_selector);
7598 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7599
7600 // Reset encoder for new configuration to take effect.
7601 ConfigureEncoder(video_encoder_config_.Copy());
7602
7603 EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(640, 480)))
7604 .WillOnce(Return(absl::nullopt));
7605 EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(320, 240)))
7606 .WillOnce(Return(SdpVideoFormat("AV1")));
7607 EXPECT_CALL(switch_callback,
7608 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7609 /*allow_default_fallback=*/false));
7610
7611 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7612 /*target_bitrate=*/DataRate::KilobitsPerSec(800),
7613 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(1000),
7614 /*link_allocation=*/DataRate::KilobitsPerSec(1000),
7615 /*fraction_lost=*/0,
7616 /*round_trip_time_ms=*/0,
7617 /*cwnd_reduce_ratio=*/0);
7618
7619 video_source_.IncomingCapturedFrame(CreateFrame(1, 640, 480));
7620 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 480));
7621 video_source_.IncomingCapturedFrame(CreateFrame(3, 320, 240));
7622
7623 AdvanceTime(TimeDelta::Zero());
7624
7625 video_stream_encoder_->Stop();
7626}
7627
philipel9b058032020-02-10 11:30:00 +01007628TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7629 constexpr int kSufficientBitrateToNotDrop = 1000;
7630 constexpr int kDontCare = 100;
7631
7632 NiceMock<MockVideoEncoder> video_encoder;
7633 NiceMock<MockEncoderSelector> encoder_selector;
7634 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7635 video_send_config_.encoder_settings.encoder_switch_request_callback =
7636 &switch_callback;
7637 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7638 &video_encoder, &encoder_selector);
7639 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7640
7641 // Reset encoder for new configuration to take effect.
7642 ConfigureEncoder(video_encoder_config_.Copy());
7643
7644 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7645 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7646 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007647 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007648 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7649 /*stable_target_bitrate=*/
7650 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7651 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007652 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007653 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007654 /*cwnd_reduce_ratio=*/0);
7655
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007656 ON_CALL(video_encoder, Encode)
philipel9b058032020-02-10 11:30:00 +01007657 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007658 ON_CALL(encoder_selector, OnEncoderBroken)
philipel9b058032020-02-10 11:30:00 +01007659 .WillByDefault(Return(SdpVideoFormat("AV2")));
7660
7661 rtc::Event encode_attempted;
7662 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007663 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7664 /*allow_default_fallback=*/true))
7665 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
philipel9b058032020-02-10 11:30:00 +01007666
7667 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
Markus Handell2cfc1af2022-08-19 08:16:48 +00007668 encode_attempted.Wait(TimeDelta::Seconds(3));
philipel9b058032020-02-10 11:30:00 +01007669
Markus Handell28c71802021-11-08 10:11:55 +01007670 AdvanceTime(TimeDelta::Zero());
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007671
philipel9b058032020-02-10 11:30:00 +01007672 video_stream_encoder_->Stop();
7673
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007674 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7675 // to it's factory, so in order for the encoder instance in the
7676 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7677 // reset the `video_stream_encoder_` here.
7678 video_stream_encoder_.reset();
7679}
7680
7681TEST_F(VideoStreamEncoderTest, SwitchEncoderOnInitFailureWithEncoderSelector) {
7682 NiceMock<MockVideoEncoder> video_encoder;
7683 NiceMock<MockEncoderSelector> encoder_selector;
7684 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7685 video_send_config_.encoder_settings.encoder_switch_request_callback =
7686 &switch_callback;
7687 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7688 &video_encoder, &encoder_selector);
7689 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7690
7691 // Reset encoder for new configuration to take effect.
7692 ConfigureEncoder(video_encoder_config_.Copy());
7693
7694 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7695 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7696 /*round_trip_time_ms=*/0,
7697 /*cwnd_reduce_ratio=*/0);
7698 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7699
7700 ON_CALL(video_encoder, InitEncode(_, _))
7701 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7702 ON_CALL(encoder_selector, OnEncoderBroken)
7703 .WillByDefault(Return(SdpVideoFormat("AV2")));
7704
7705 rtc::Event encode_attempted;
7706 EXPECT_CALL(switch_callback,
7707 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7708 /*allow_default_fallback=*/true))
7709 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7710
7711 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
Markus Handell2cfc1af2022-08-19 08:16:48 +00007712 encode_attempted.Wait(TimeDelta::Seconds(3));
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007713
7714 AdvanceTime(TimeDelta::Zero());
7715
7716 video_stream_encoder_->Stop();
7717
7718 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7719 // to it's factory, so in order for the encoder instance in the
7720 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7721 // reset the `video_stream_encoder_` here.
7722 video_stream_encoder_.reset();
7723}
7724
7725TEST_F(VideoStreamEncoderTest,
7726 SwitchEncoderOnInitFailureWithoutEncoderSelector) {
7727 NiceMock<MockVideoEncoder> video_encoder;
7728 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7729 video_send_config_.encoder_settings.encoder_switch_request_callback =
7730 &switch_callback;
7731 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7732 &video_encoder, /*encoder_selector=*/nullptr);
7733 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7734
7735 // Reset encoder for new configuration to take effect.
7736 ConfigureEncoder(video_encoder_config_.Copy());
7737
7738 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7739 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7740 /*round_trip_time_ms=*/0,
7741 /*cwnd_reduce_ratio=*/0);
7742 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7743
7744 ON_CALL(video_encoder, InitEncode(_, _))
7745 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7746
7747 rtc::Event encode_attempted;
7748 EXPECT_CALL(switch_callback,
7749 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "VP8"),
7750 /*allow_default_fallback=*/true))
7751 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7752
7753 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
Markus Handell2cfc1af2022-08-19 08:16:48 +00007754 encode_attempted.Wait(TimeDelta::Seconds(3));
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007755
7756 AdvanceTime(TimeDelta::Zero());
7757
7758 video_stream_encoder_->Stop();
7759
7760 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007761 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007762 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7763 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007764 video_stream_encoder_.reset();
7765}
7766
Byoungchan Lee13fe3672022-04-06 10:44:42 +09007767TEST_F(VideoStreamEncoderTest, NullEncoderReturnSwitch) {
7768 // As a variant of EncoderSelectorBrokenEncoderSwitch, when a null
7769 // VideoEncoder is passed in encoder_factory, it checks whether
7770 // Codec Switch occurs without a crash.
7771 constexpr int kSufficientBitrateToNotDrop = 1000;
7772 constexpr int kDontCare = 100;
7773
7774 NiceMock<MockEncoderSelector> encoder_selector;
7775 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7776 video_send_config_.encoder_settings.encoder_switch_request_callback =
7777 &switch_callback;
7778 auto encoder_factory =
7779 std::make_unique<test::VideoEncoderNullableProxyFactory>(
7780 /*encoder=*/nullptr, &encoder_selector);
7781 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7782
7783 // Reset encoder for new configuration to take effect.
7784 ConfigureEncoder(video_encoder_config_.Copy());
7785 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7786 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7787 // not fail.
7788 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7789 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7790 /*stable_target_bitrate=*/
7791 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7792 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7793 /*fraction_lost=*/0,
7794 /*round_trip_time_ms=*/0,
7795 /*cwnd_reduce_ratio=*/0);
7796 ON_CALL(encoder_selector, OnEncoderBroken)
7797 .WillByDefault(Return(SdpVideoFormat("AV2")));
7798 rtc::Event encode_attempted;
7799 EXPECT_CALL(switch_callback,
7800 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7801 /*allow_default_fallback=*/_))
7802 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7803
7804 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
Markus Handell2cfc1af2022-08-19 08:16:48 +00007805 encode_attempted.Wait(TimeDelta::Seconds(3));
Byoungchan Lee13fe3672022-04-06 10:44:42 +09007806
7807 AdvanceTime(TimeDelta::Zero());
7808
7809 video_stream_encoder_->Stop();
7810
7811 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7812 // to it's factory, so in order for the encoder instance in the
7813 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7814 // reset the `video_stream_encoder_` here.
7815 video_stream_encoder_.reset();
7816}
7817
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007818TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007819 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007820 const int kFrameWidth = 320;
7821 const int kFrameHeight = 180;
7822
7823 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007824 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007825 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007826 /*target_bitrate=*/rate,
7827 /*stable_target_bitrate=*/rate,
7828 /*link_allocation=*/rate,
7829 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007830 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007831 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007832
7833 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007834 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007835 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7836 frame.set_rotation(kVideoRotation_270);
7837 video_source_.IncomingCapturedFrame(frame);
7838 WaitForEncodedFrame(timestamp_ms);
7839 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7840
7841 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007842 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007843 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007844 /*target_bitrate=*/new_stable_rate,
7845 /*stable_target_bitrate=*/new_stable_rate,
7846 /*link_allocation=*/rate,
7847 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007848 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007849 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007850 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7851 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7852 video_stream_encoder_->Stop();
7853}
7854
7855TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007856 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007857 const int kFrameWidth = 320;
7858 const int kFrameHeight = 180;
7859
7860 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007861 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007862 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007863 /*target_bitrate=*/rate,
7864 /*stable_target_bitrate=*/rate,
7865 /*link_allocation=*/rate,
7866 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007867 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007868 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007869
7870 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007871 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007872 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7873 frame.set_rotation(kVideoRotation_270);
7874 video_source_.IncomingCapturedFrame(frame);
7875 WaitForEncodedFrame(timestamp_ms);
7876 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7877
7878 // Set a higher target rate without changing the link_allocation. Should not
7879 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007880 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007881 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007882 /*target_bitrate=*/rate,
7883 /*stable_target_bitrate=*/new_stable_rate,
7884 /*link_allocation=*/rate,
7885 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007886 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007887 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007888 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7889 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7890 video_stream_encoder_->Stop();
7891}
7892
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007893TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01007894 test::ScopedKeyValueConfig field_trials(
7895 field_trials_,
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007896 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7897 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7898 const int kFramerateFps = 30;
7899 const int kWidth = 1920;
7900 const int kHeight = 1080;
7901 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7902 // Works on screenshare mode.
7903 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7904 // We rely on the automatic resolution adaptation, but we handle framerate
7905 // adaptation manually by mocking the stats proxy.
7906 video_source_.set_adaptation_enabled(true);
7907
7908 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007909 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007910 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007911 video_stream_encoder_->SetSource(&video_source_,
7912 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007913 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007914
7915 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7916 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7917
7918 // Pass enough frames with the full update to trigger animation detection.
7919 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007920 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007921 frame.set_ntp_time_ms(timestamp_ms);
7922 frame.set_timestamp_us(timestamp_ms * 1000);
7923 video_source_.IncomingCapturedFrame(frame);
7924 WaitForEncodedFrame(timestamp_ms);
7925 }
7926
7927 // Resolution should be limited.
7928 rtc::VideoSinkWants expected;
7929 expected.max_framerate_fps = kFramerateFps;
7930 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007931 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007932
7933 // Pass one frame with no known update.
7934 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007935 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007936 frame.set_ntp_time_ms(timestamp_ms);
7937 frame.set_timestamp_us(timestamp_ms * 1000);
7938 frame.clear_update_rect();
7939
7940 video_source_.IncomingCapturedFrame(frame);
7941 WaitForEncodedFrame(timestamp_ms);
7942
7943 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007944 EXPECT_THAT(video_source_.sink_wants(),
7945 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007946
7947 video_stream_encoder_->Stop();
7948}
7949
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007950TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7951 const int kWidth = 720; // 540p adapted down.
7952 const int kHeight = 405;
7953 const int kNumFrames = 3;
7954 // Works on screenshare mode.
7955 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7956 /*num_spatial_layers=*/2, /*screenshare=*/true);
7957
7958 video_source_.set_adaptation_enabled(true);
7959
7960 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007961 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007962
7963 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7964
7965 // Pass enough frames with the full update to trigger animation detection.
7966 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007967 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007968 frame.set_ntp_time_ms(timestamp_ms);
7969 frame.set_timestamp_us(timestamp_ms * 1000);
7970 video_source_.IncomingCapturedFrame(frame);
7971 WaitForEncodedFrame(timestamp_ms);
7972 }
7973
7974 video_stream_encoder_->Stop();
7975}
7976
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007977TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7978 const float downscale_factors[] = {4.0, 2.0, 1.0};
7979 const int number_layers =
7980 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7981 VideoEncoderConfig config;
Jonas Oreland80c87d72022-09-29 15:01:09 +02007982 webrtc::VideoEncoder::EncoderInfo encoder_info;
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007983 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7984 for (int i = 0; i < number_layers; ++i) {
7985 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7986 config.simulcast_layers[i].active = true;
7987 }
7988 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007989 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007990 "VP8", /*max qp*/ 56, /*screencast*/ false,
Jonas Oreland80c87d72022-09-29 15:01:09 +02007991 /*screenshare enabled*/ false, encoder_info);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007992 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007993 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7994 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007995
7996 // First initialization.
7997 // Encoder should be initialized. Next frame should be key frame.
7998 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7999 sink_.SetNumExpectedLayers(number_layers);
8000 int64_t timestamp_ms = kFrameIntervalMs;
8001 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8002 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008003 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008004 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8005 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
8006 VideoFrameType::kVideoFrameKey,
8007 VideoFrameType::kVideoFrameKey}));
8008
8009 // Disable top layer.
8010 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
8011 config.simulcast_layers[number_layers - 1].active = false;
8012 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8013 sink_.SetNumExpectedLayers(number_layers - 1);
8014 timestamp_ms += kFrameIntervalMs;
8015 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8016 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008017 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008018 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8019 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
8020 VideoFrameType::kVideoFrameDelta,
8021 VideoFrameType::kVideoFrameDelta}));
8022
8023 // Re-enable top layer.
8024 // Encoder should be re-initialized. Next frame should be key frame.
8025 config.simulcast_layers[number_layers - 1].active = true;
8026 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8027 sink_.SetNumExpectedLayers(number_layers);
8028 timestamp_ms += kFrameIntervalMs;
8029 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8030 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008031 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008032 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8033 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
8034 VideoFrameType::kVideoFrameKey,
8035 VideoFrameType::kVideoFrameKey}));
8036
8037 // Top layer max rate change.
8038 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
8039 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
8040 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8041 sink_.SetNumExpectedLayers(number_layers);
8042 timestamp_ms += kFrameIntervalMs;
8043 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8044 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008045 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008046 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8047 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
8048 VideoFrameType::kVideoFrameDelta,
8049 VideoFrameType::kVideoFrameDelta}));
8050
8051 // Top layer resolution change.
8052 // Encoder should be re-initialized. Next frame should be key frame.
8053 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
8054 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8055 sink_.SetNumExpectedLayers(number_layers);
8056 timestamp_ms += kFrameIntervalMs;
8057 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8058 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008059 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008060 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8061 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
8062 VideoFrameType::kVideoFrameKey,
8063 VideoFrameType::kVideoFrameKey}));
8064 video_stream_encoder_->Stop();
8065}
8066
Henrik Boström1124ed12021-02-25 10:30:39 +01008067TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
8068 const int kFrameWidth = 1280;
8069 const int kFrameHeight = 720;
8070
8071 SetUp();
8072 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008073 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01008074
8075 // Capturing a frame should reconfigure the encoder and expose the encoder
8076 // resolution, which is the same as the input frame.
8077 int64_t timestamp_ms = kFrameIntervalMs;
8078 video_source_.IncomingCapturedFrame(
8079 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8080 WaitForEncodedFrame(timestamp_ms);
8081 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8082 EXPECT_THAT(video_source_.sink_wants().resolutions,
8083 ::testing::ElementsAreArray(
8084 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
8085
8086 video_stream_encoder_->Stop();
8087}
8088
8089TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
8090 // Pick downscale factors such that we never encode at full resolution - this
8091 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 16:22:11 +02008092 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 10:30:39 +01008093 // encoder should not ask for the frame resolution. This allows video frames
8094 // to have the appearence of one resolution but optimize its internal buffers
8095 // for what is actually encoded.
8096 const size_t kNumSimulcastLayers = 3u;
8097 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
8098 const int kFrameWidth = 1280;
8099 const int kFrameHeight = 720;
8100 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8101 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8102 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8103 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8104 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8105 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8106
8107 VideoEncoderConfig config;
Jonas Oreland80c87d72022-09-29 15:01:09 +02008108 webrtc::VideoEncoder::EncoderInfo encoder_info;
Henrik Boström1124ed12021-02-25 10:30:39 +01008109 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
8110 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
8111 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
8112 config.simulcast_layers[i].active = true;
8113 }
8114 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02008115 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01008116 "VP8", /*max qp*/ 56, /*screencast*/ false,
Jonas Oreland80c87d72022-09-29 15:01:09 +02008117 /*screenshare enabled*/ false, encoder_info);
Henrik Boström1124ed12021-02-25 10:30:39 +01008118 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008119 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8120 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01008121
8122 // Capture a frame with all layers active.
8123 int64_t timestamp_ms = kFrameIntervalMs;
8124 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
8125 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8126 video_source_.IncomingCapturedFrame(
8127 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8128 WaitForEncodedFrame(timestamp_ms);
8129 // Expect encoded resolutions to match the expected simulcast layers.
8130 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8131 EXPECT_THAT(
8132 video_source_.sink_wants().resolutions,
8133 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
8134
8135 // Capture a frame with one of the layers inactive.
8136 timestamp_ms += kFrameIntervalMs;
8137 config.simulcast_layers[2].active = false;
8138 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
8139 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8140 video_source_.IncomingCapturedFrame(
8141 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8142 WaitForEncodedFrame(timestamp_ms);
8143
8144 // Expect encoded resolutions to match the expected simulcast layers.
8145 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8146 EXPECT_THAT(video_source_.sink_wants().resolutions,
8147 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
8148
8149 // Capture a frame with all but one layer turned off.
8150 timestamp_ms += kFrameIntervalMs;
8151 config.simulcast_layers[1].active = false;
8152 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
8153 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8154 video_source_.IncomingCapturedFrame(
8155 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8156 WaitForEncodedFrame(timestamp_ms);
8157
8158 // Expect encoded resolutions to match the expected simulcast layers.
8159 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8160 EXPECT_THAT(video_source_.sink_wants().resolutions,
8161 ::testing::ElementsAreArray({kLayer0Size}));
8162
8163 video_stream_encoder_->Stop();
8164}
8165
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008166TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008167 ResetEncoder("VP8", 1, 1, 1, false);
8168
Niels Möller8b692902021-06-14 12:04:57 +02008169 // Force encoder reconfig.
8170 video_source_.IncomingCapturedFrame(
8171 CreateFrame(1, codec_width_, codec_height_));
8172 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8173
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008174 // Set QP on encoded frame and pass the frame to encode complete callback.
8175 // Since QP is present QP parsing won't be triggered and the original value
8176 // should be kept.
8177 EncodedImage encoded_image;
8178 encoded_image.qp_ = 123;
8179 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8180 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8181 CodecSpecificInfo codec_info;
8182 codec_info.codecType = kVideoCodecVP8;
8183 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
Markus Handell2cfc1af2022-08-19 08:16:48 +00008184 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeout));
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008185 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
8186 video_stream_encoder_->Stop();
8187}
8188
8189TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008190 ResetEncoder("VP8", 1, 1, 1, false);
8191
Niels Möller8b692902021-06-14 12:04:57 +02008192 // Force encoder reconfig.
8193 video_source_.IncomingCapturedFrame(
8194 CreateFrame(1, codec_width_, codec_height_));
8195 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8196
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008197 // Pass an encoded frame without QP to encode complete callback. QP should be
8198 // parsed and set.
8199 EncodedImage encoded_image;
8200 encoded_image.qp_ = -1;
8201 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8202 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8203 CodecSpecificInfo codec_info;
8204 codec_info.codecType = kVideoCodecVP8;
8205 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
Markus Handell2cfc1af2022-08-19 08:16:48 +00008206 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeout));
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008207 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
8208 video_stream_encoder_->Stop();
8209}
8210
8211TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01008212 webrtc::test::ScopedKeyValueConfig field_trials(
8213 field_trials_, "WebRTC-QpParsingKillSwitch/Enabled/");
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008214
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008215 ResetEncoder("VP8", 1, 1, 1, false);
8216
Niels Möller8b692902021-06-14 12:04:57 +02008217 // Force encoder reconfig.
8218 video_source_.IncomingCapturedFrame(
8219 CreateFrame(1, codec_width_, codec_height_));
8220 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8221
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008222 EncodedImage encoded_image;
8223 encoded_image.qp_ = -1;
8224 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8225 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8226 CodecSpecificInfo codec_info;
8227 codec_info.codecType = kVideoCodecVP8;
8228 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
Markus Handell2cfc1af2022-08-19 08:16:48 +00008229 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeout));
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008230 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
8231 video_stream_encoder_->Stop();
8232}
8233
Sergey Silkind19e3b92021-03-16 10:05:30 +00008234TEST_F(VideoStreamEncoderTest,
8235 QualityScalingNotAllowed_QualityScalingDisabled) {
8236 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8237
8238 // Disable scaling settings in encoder info.
8239 fake_encoder_.SetQualityScaling(false);
8240 // Disable quality scaling in encoder config.
8241 video_encoder_config.is_quality_scaling_allowed = false;
8242 ConfigureEncoder(std::move(video_encoder_config));
8243
8244 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008245 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008246
8247 test::FrameForwarder source;
8248 video_stream_encoder_->SetSource(
8249 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8250 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8251 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8252
8253 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8254 WaitForEncodedFrame(1);
8255 video_stream_encoder_->TriggerQualityLow();
8256 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8257
8258 video_stream_encoder_->Stop();
8259}
8260
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008261TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
8262 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8263
8264 // Disable scaling settings in encoder info.
8265 fake_encoder_.SetQualityScaling(false);
8266 // Set QP trusted in encoder info.
8267 fake_encoder_.SetIsQpTrusted(true);
8268 // Enable quality scaling in encoder config.
8269 video_encoder_config.is_quality_scaling_allowed = false;
8270 ConfigureEncoder(std::move(video_encoder_config));
8271
8272 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008273 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008274
8275 test::FrameForwarder source;
8276 video_stream_encoder_->SetSource(
8277 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8278 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8279 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8280
8281 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8282 WaitForEncodedFrame(1);
8283 video_stream_encoder_->TriggerQualityLow();
8284 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8285
8286 video_stream_encoder_->Stop();
8287}
8288
Shuhai Pengf2707702021-09-29 17:19:44 +08008289TEST_F(VideoStreamEncoderTest,
8290 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8291 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8292
8293 // Disable scaling settings in encoder info.
8294 fake_encoder_.SetQualityScaling(false);
8295 // Set QP trusted in encoder info.
8296 fake_encoder_.SetIsQpTrusted(true);
8297 // Enable quality scaling in encoder config.
8298 video_encoder_config.is_quality_scaling_allowed = false;
8299 ConfigureEncoder(std::move(video_encoder_config));
8300
8301 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008302 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008303
8304 test::FrameForwarder source;
8305 video_stream_encoder_->SetSource(
8306 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8307 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8308 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8309
8310 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8311 WaitForEncodedFrame(1);
8312 video_stream_encoder_->TriggerQualityLow();
8313 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8314
8315 video_stream_encoder_->Stop();
8316}
8317
8318TEST_F(VideoStreamEncoderTest,
8319 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8320 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8321
8322 // Disable scaling settings in encoder info.
8323 fake_encoder_.SetQualityScaling(false);
8324 // Set QP trusted in encoder info.
8325 fake_encoder_.SetIsQpTrusted(false);
8326 // Enable quality scaling in encoder config.
8327 video_encoder_config.is_quality_scaling_allowed = false;
8328 ConfigureEncoder(std::move(video_encoder_config));
8329
8330 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008331 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008332
8333 test::FrameForwarder source;
8334 video_stream_encoder_->SetSource(
8335 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8336 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8337 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8338
8339 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8340 WaitForEncodedFrame(1);
8341 video_stream_encoder_->TriggerQualityLow();
8342 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8343
8344 video_stream_encoder_->Stop();
8345}
8346
8347TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8348 // Set QP trusted in encoder info.
8349 fake_encoder_.SetIsQpTrusted(false);
8350
8351 const int MinEncBitrateKbps = 30;
8352 const int MaxEncBitrateKbps = 100;
8353 const int MinStartBitrateKbp = 50;
8354 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8355 /*frame_size_pixels=*/codec_width_ * codec_height_,
8356 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8357 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8358 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8359
8360 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008361 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008362
8363 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8364
8365 VideoEncoderConfig video_encoder_config;
8366 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8367 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8368 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8369 MinEncBitrateKbps * 1000;
8370 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8371 kMaxPayloadLength);
8372
8373 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8374 WaitForEncodedFrame(1);
8375 EXPECT_EQ(
8376 MaxEncBitrateKbps,
8377 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8378 EXPECT_EQ(
8379 MinEncBitrateKbps,
8380 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8381
8382 video_stream_encoder_->Stop();
8383}
8384
8385TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8386 // Set QP trusted in encoder info.
8387 fake_encoder_.SetIsQpTrusted(false);
8388
8389 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8390 EncoderInfoSettings::
8391 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8392 codec_width_ * codec_height_,
8393 EncoderInfoSettings::
8394 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8395 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8396
8397 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8398 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8399 const int TargetEncBitrate = MaxEncBitrate;
8400 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8401 DataRate::BitsPerSec(TargetEncBitrate),
8402 DataRate::BitsPerSec(TargetEncBitrate),
8403 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8404
8405 VideoEncoderConfig video_encoder_config;
8406 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8407 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8408 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8409 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8410 kMaxPayloadLength);
8411
8412 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8413 WaitForEncodedFrame(1);
8414 EXPECT_EQ(
8415 MaxEncBitrate / 1000,
8416 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8417 EXPECT_EQ(
8418 MinEncBitrate / 1000,
8419 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8420
8421 video_stream_encoder_->Stop();
8422}
8423
Erik Språnge4589cb2022-04-06 16:44:30 +02008424TEST_F(VideoStreamEncoderTest, NormalComplexityWithMoreThanTwoCores) {
8425 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8426 /*num_spatial_layers=*/1,
8427 /*screenshare=*/false, /*allocation_callback_type=*/
8428 VideoStreamEncoder::BitrateAllocationCallbackType::
8429 kVideoBitrateAllocationWhenScreenSharing,
8430 /*num_cores=*/3);
8431
8432 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8433 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8434 video_source_.IncomingCapturedFrame(
8435 CreateFrame(1, /*width=*/320, /*height=*/180));
8436 WaitForEncodedFrame(1);
8437 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8438 VideoCodecComplexity::kComplexityNormal);
8439 video_stream_encoder_->Stop();
8440}
8441
8442TEST_F(VideoStreamEncoderTest,
8443 NormalComplexityWhenLowTierOptimizationsAreDisabled) {
8444 webrtc::test::ScopedKeyValueConfig field_trials(
8445 field_trials_, "WebRTC-VP9-LowTierOptimizations/Disabled/");
8446
8447 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8448 /*num_spatial_layers=*/1,
8449 /*screenshare=*/false, /*allocation_callback_type=*/
8450 VideoStreamEncoder::BitrateAllocationCallbackType::
8451 kVideoBitrateAllocationWhenScreenSharing,
8452 /*num_cores=*/2);
8453
8454 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8455 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8456 video_source_.IncomingCapturedFrame(
8457 CreateFrame(1, /*width=*/320, /*height=*/180));
8458 WaitForEncodedFrame(1);
8459 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8460 VideoCodecComplexity::kComplexityNormal);
8461 video_stream_encoder_->Stop();
8462}
8463
8464TEST_F(VideoStreamEncoderTest, LowComplexityWithTwoCores) {
8465 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8466 /*num_spatial_layers=*/1,
8467 /*screenshare=*/false, /*allocation_callback_type=*/
8468 VideoStreamEncoder::BitrateAllocationCallbackType::
8469 kVideoBitrateAllocationWhenScreenSharing,
8470 /*num_cores=*/2);
8471
8472 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8473 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8474 video_source_.IncomingCapturedFrame(
8475 CreateFrame(1, /*width=*/320, /*height=*/180));
8476 WaitForEncodedFrame(1);
8477 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8478 VideoCodecComplexity::kComplexityLow);
8479 video_stream_encoder_->Stop();
8480}
8481
Sergey Silkind19e3b92021-03-16 10:05:30 +00008482#if !defined(WEBRTC_IOS)
8483// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8484// disabled by default on iOS.
8485TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8486 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8487
8488 // Disable scaling settings in encoder info.
8489 fake_encoder_.SetQualityScaling(false);
8490 // Enable quality scaling in encoder config.
8491 video_encoder_config.is_quality_scaling_allowed = true;
8492 ConfigureEncoder(std::move(video_encoder_config));
8493
8494 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008495 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008496
8497 test::FrameForwarder source;
8498 video_stream_encoder_->SetSource(
8499 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8500 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8501 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8502
8503 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8504 WaitForEncodedFrame(1);
8505 video_stream_encoder_->TriggerQualityLow();
8506 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8507
8508 video_stream_encoder_->Stop();
8509}
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008510
8511TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8512 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8513
8514 // Disable scaling settings in encoder info.
8515 fake_encoder_.SetQualityScaling(false);
8516 // Set QP trusted in encoder info.
8517 fake_encoder_.SetIsQpTrusted(true);
8518 // Enable quality scaling in encoder config.
8519 video_encoder_config.is_quality_scaling_allowed = true;
8520 ConfigureEncoder(std::move(video_encoder_config));
8521
8522 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008523 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008524
8525 test::FrameForwarder source;
8526 video_stream_encoder_->SetSource(
8527 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8528 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8529 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8530
8531 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8532 WaitForEncodedFrame(1);
8533 video_stream_encoder_->TriggerQualityLow();
8534 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8535
8536 video_stream_encoder_->Stop();
8537}
Shuhai Pengf2707702021-09-29 17:19:44 +08008538
8539TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8540 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8541
8542 // Disable scaling settings in encoder info.
8543 fake_encoder_.SetQualityScaling(false);
8544 // Set QP not trusted in encoder info.
8545 fake_encoder_.SetIsQpTrusted(false);
8546 // Enable quality scaling in encoder config.
8547 video_encoder_config.is_quality_scaling_allowed = true;
8548 ConfigureEncoder(std::move(video_encoder_config));
8549
8550 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008551 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008552
8553 test::FrameForwarder source;
8554 video_stream_encoder_->SetSource(
8555 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8556 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8557 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8558
8559 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8560 WaitForEncodedFrame(1);
8561 video_stream_encoder_->TriggerQualityLow();
8562 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8563 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8564 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8565
8566 video_stream_encoder_->Stop();
8567}
8568
8569TEST_F(VideoStreamEncoderTest,
8570 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8571 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8572
8573 // Disable scaling settings in encoder info.
8574 fake_encoder_.SetQualityScaling(false);
8575 // Set QP trusted in encoder info.
8576 fake_encoder_.SetIsQpTrusted(true);
8577 // Enable quality scaling in encoder config.
8578 video_encoder_config.is_quality_scaling_allowed = true;
8579 ConfigureEncoder(std::move(video_encoder_config));
8580
8581 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008582 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008583
8584 test::FrameForwarder source;
8585 video_stream_encoder_->SetSource(
8586 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8587 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8588 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8589
8590 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8591 WaitForEncodedFrame(1);
8592 video_stream_encoder_->TriggerQualityLow();
8593 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8594 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8595
8596 video_stream_encoder_->Stop();
8597}
8598
8599TEST_F(VideoStreamEncoderTest,
8600 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8601 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8602
8603 // Disable scaling settings in encoder info.
8604 fake_encoder_.SetQualityScaling(false);
8605 // Set QP trusted in encoder info.
8606 fake_encoder_.SetIsQpTrusted(false);
8607 // Enable quality scaling in encoder config.
8608 video_encoder_config.is_quality_scaling_allowed = true;
8609 ConfigureEncoder(std::move(video_encoder_config));
8610
8611 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008612 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008613
8614 test::FrameForwarder source;
8615 video_stream_encoder_->SetSource(
8616 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8617 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8618 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8619
8620 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8621 WaitForEncodedFrame(1);
8622 video_stream_encoder_->TriggerQualityLow();
8623 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8624
8625 video_stream_encoder_->Stop();
8626}
8627
Erik Språng5e13d052022-08-02 11:42:49 +02008628TEST_F(VideoStreamEncoderTest,
8629 RequestsRefreshFrameAfterEarlyDroppedNativeFrame) {
8630 // Send a native frame before encoder rates have been set. The encoder is
8631 // seen as paused at this time.
8632 rtc::Event frame_destroyed_event;
8633 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
8634 /*ntp_time_ms=*/1, &frame_destroyed_event, codec_width_, codec_height_));
8635
8636 // Frame should be dropped and destroyed.
8637 ExpectDroppedFrame();
Markus Handell2cfc1af2022-08-19 08:16:48 +00008638 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
Erik Språng5e13d052022-08-02 11:42:49 +02008639 EXPECT_EQ(video_source_.refresh_frames_requested_, 0);
8640
8641 // Set bitrates, unpausing the encoder and triggering a request for a refresh
8642 // frame.
8643 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8644 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8645 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8646 EXPECT_EQ(video_source_.refresh_frames_requested_, 1);
8647
8648 video_stream_encoder_->Stop();
8649}
8650
Erik Språnge4589cb2022-04-06 16:44:30 +02008651#endif // !defined(WEBRTC_IOS)
Sergey Silkind19e3b92021-03-16 10:05:30 +00008652
Henrik Boström56db9ff2021-03-24 09:06:45 +01008653// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8654class VideoStreamEncoderWithRealEncoderTest
8655 : public VideoStreamEncoderTest,
8656 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8657 public:
8658 VideoStreamEncoderWithRealEncoderTest()
8659 : VideoStreamEncoderTest(),
8660 codec_type_(std::get<0>(GetParam())),
8661 allow_i420_conversion_(std::get<1>(GetParam())) {}
8662
8663 void SetUp() override {
8664 VideoStreamEncoderTest::SetUp();
8665 std::unique_ptr<VideoEncoder> encoder;
8666 switch (codec_type_) {
8667 case kVideoCodecVP8:
8668 encoder = VP8Encoder::Create();
8669 break;
8670 case kVideoCodecVP9:
8671 encoder = VP9Encoder::Create();
8672 break;
8673 case kVideoCodecAV1:
philipel09a28482022-05-25 09:47:06 +02008674 encoder = CreateLibaomAv1Encoder();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008675 break;
8676 case kVideoCodecH264:
8677 encoder =
8678 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8679 break;
8680 case kVideoCodecMultiplex:
8681 mock_encoder_factory_for_multiplex_ =
8682 std::make_unique<MockVideoEncoderFactory>();
8683 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8684 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8685 .WillRepeatedly([] { return VP8Encoder::Create(); });
8686 encoder = std::make_unique<MultiplexEncoderAdapter>(
8687 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8688 false);
8689 break;
8690 default:
Artem Titovd3251962021-11-15 16:57:07 +01008691 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008692 }
8693 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8694 }
8695
8696 void TearDown() override {
8697 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 16:22:11 +02008698 // Ensure `video_stream_encoder_` is destroyed before
8699 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 09:06:45 +01008700 video_stream_encoder_.reset();
8701 VideoStreamEncoderTest::TearDown();
8702 }
8703
8704 protected:
8705 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8706 std::unique_ptr<VideoEncoder> encoder) {
8707 // Configure VSE to use the encoder.
8708 encoder_ = std::move(encoder);
8709 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8710 encoder_.get(), &encoder_selector_);
8711 video_send_config_.encoder_settings.encoder_factory =
8712 encoder_proxy_factory_.get();
8713 VideoEncoderConfig video_encoder_config;
8714 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8715 video_encoder_config_ = video_encoder_config.Copy();
8716 ConfigureEncoder(video_encoder_config_.Copy());
8717
8718 // Set bitrate to ensure frame is not dropped.
8719 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008720 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008721 }
8722
8723 const VideoCodecType codec_type_;
8724 const bool allow_i420_conversion_;
8725 NiceMock<MockEncoderSelector> encoder_selector_;
8726 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8727 std::unique_ptr<VideoEncoder> encoder_;
8728 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8729};
8730
8731TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8732 auto native_i420_frame = test::CreateMappableNativeFrame(
8733 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8734 video_source_.IncomingCapturedFrame(native_i420_frame);
8735 WaitForEncodedFrame(codec_width_, codec_height_);
8736
8737 auto mappable_native_buffer =
8738 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8739 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8740 mappable_native_buffer->GetMappedFramedBuffers();
8741 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8742 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8743 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8744 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8745}
8746
8747TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8748 auto native_nv12_frame = test::CreateMappableNativeFrame(
8749 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8750 video_source_.IncomingCapturedFrame(native_nv12_frame);
8751 WaitForEncodedFrame(codec_width_, codec_height_);
8752
8753 auto mappable_native_buffer =
8754 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8755 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8756 mappable_native_buffer->GetMappedFramedBuffers();
8757 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8758 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8759 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8760 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8761
8762 if (!allow_i420_conversion_) {
8763 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8764 }
8765}
8766
Erik Språng7444b192021-06-02 14:02:13 +02008767TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8768 if (codec_type_ == kVideoCodecMultiplex) {
8769 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8770 return;
8771 }
8772
8773 const size_t kNumSpatialLayers = 3u;
8774 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8775 const int kFrameWidth = 1280;
8776 const int kFrameHeight = 720;
8777 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8778 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8779 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8780 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8781 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8782 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8783
8784 VideoEncoderConfig config;
Jonas Oreland80c87d72022-09-29 15:01:09 +02008785 webrtc::VideoEncoder::EncoderInfo encoder_info;
Erik Språng7444b192021-06-02 14:02:13 +02008786 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8787 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008788 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008789 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8790 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8791 vp9_settings.numberOfTemporalLayers = 3;
8792 vp9_settings.automaticResizeOn = false;
8793 config.encoder_specific_settings =
8794 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8795 vp9_settings);
8796 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8797 /*fps=*/30.0,
8798 /*first_active_layer=*/0,
8799 /*num_spatial_layers=*/3,
8800 /*num_temporal_layers=*/3,
8801 /*is_screenshare=*/false);
8802 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8803 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008804 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008805 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8806 /*fps=*/30.0,
8807 /*first_active_layer=*/0,
8808 /*num_spatial_layers=*/3,
8809 /*num_temporal_layers=*/3,
8810 /*is_screenshare=*/false);
Niels Möller79d566b2022-04-29 11:03:13 +02008811 config.simulcast_layers[0].scalability_mode = ScalabilityMode::kL3T3_KEY;
Erik Språng7444b192021-06-02 14:02:13 +02008812 } else {
8813 // Simulcast for VP8/H264.
8814 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8815 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8816 config.simulcast_layers[i].scale_resolution_down_by =
8817 kDownscaleFactors[i];
8818 config.simulcast_layers[i].active = true;
8819 }
8820 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8821 // Turn off frame dropping to prevent flakiness.
Niels Möller807328f2022-05-12 16:16:39 +02008822 config.frame_drop_enabled = false;
Erik Språng7444b192021-06-02 14:02:13 +02008823 }
8824 }
8825
8826 auto set_layer_active = [&](int layer_idx, bool active) {
8827 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8828 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8829 config.spatial_layers[layer_idx].active = active;
8830 } else {
8831 config.simulcast_layers[layer_idx].active = active;
8832 }
8833 };
8834
8835 config.video_stream_factory =
8836 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8837 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8838 /*screencast*/ false,
Jonas Oreland80c87d72022-09-29 15:01:09 +02008839 /*screenshare enabled*/ false, encoder_info);
Erik Språng7444b192021-06-02 14:02:13 +02008840 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008841 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8842 0, 0, 0);
Erik Språng7444b192021-06-02 14:02:13 +02008843
8844 // Capture a frame with all layers active.
8845 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8846 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8847 int64_t timestamp_ms = kFrameIntervalMs;
8848 video_source_.IncomingCapturedFrame(
8849 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8850
8851 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8852 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8853
8854 // Capture a frame with one of the layers inactive.
8855 set_layer_active(2, false);
8856 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8857 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8858 timestamp_ms += kFrameIntervalMs;
8859 video_source_.IncomingCapturedFrame(
8860 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8861 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8862
8863 // New target bitrates signaled based on lower resolution.
8864 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8865 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8866 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8867 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8868
8869 // Re-enable the top layer.
8870 set_layer_active(2, true);
8871 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8872 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8873 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8874
8875 // Bitrate target adjusted back up to enable HD layer...
8876 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8877 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8878 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8879 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8880
8881 // ...then add a new frame.
8882 timestamp_ms += kFrameIntervalMs;
8883 video_source_.IncomingCapturedFrame(
8884 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8885 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8886 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8887
8888 video_stream_encoder_->Stop();
8889}
8890
Henrik Boström56db9ff2021-03-24 09:06:45 +01008891std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8892 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8893 VideoCodecType codec_type = std::get<0>(info.param);
8894 bool allow_i420_conversion = std::get<1>(info.param);
8895 std::string str;
8896 switch (codec_type) {
8897 case kVideoCodecGeneric:
8898 str = "Generic";
8899 break;
8900 case kVideoCodecVP8:
8901 str = "VP8";
8902 break;
8903 case kVideoCodecVP9:
8904 str = "VP9";
8905 break;
8906 case kVideoCodecAV1:
8907 str = "AV1";
8908 break;
8909 case kVideoCodecH264:
8910 str = "H264";
8911 break;
8912 case kVideoCodecMultiplex:
8913 str = "Multiplex";
8914 break;
8915 default:
Artem Titovd3251962021-11-15 16:57:07 +01008916 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008917 }
8918 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8919 return str;
8920}
8921
8922constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8923 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8924constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8925 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8926constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
Ilya Nikolaevskiy1bcdafc2022-03-09 16:01:07 +01008927 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/false);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008928constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8929 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8930#if defined(WEBRTC_USE_H264)
8931constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8932 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8933
8934// The windows compiler does not tolerate #if statements inside the
8935// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8936// and without H264).
8937INSTANTIATE_TEST_SUITE_P(
8938 All,
8939 VideoStreamEncoderWithRealEncoderTest,
8940 ::testing::Values(kVP8DisallowConversion,
8941 kVP9DisallowConversion,
8942 kAV1AllowConversion,
8943 kMultiplexDisallowConversion,
8944 kH264AllowConversion),
8945 TestParametersVideoCodecAndAllowI420ConversionToString);
8946#else
8947INSTANTIATE_TEST_SUITE_P(
8948 All,
8949 VideoStreamEncoderWithRealEncoderTest,
8950 ::testing::Values(kVP8DisallowConversion,
8951 kVP9DisallowConversion,
8952 kAV1AllowConversion,
8953 kMultiplexDisallowConversion),
8954 TestParametersVideoCodecAndAllowI420ConversionToString);
8955#endif
8956
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008957class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8958 protected:
8959 void RunTest(const std::vector<VideoStream>& configs,
8960 const int expected_num_init_encode) {
8961 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008962 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008963 InsertFrameAndWaitForEncoded();
8964 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8965 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008966 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8967 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008968
8969 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8970 ConfigureEncoder(configs[1]);
8971 InsertFrameAndWaitForEncoded();
8972 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8973 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008974 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008975 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 10:07:11 +02008976 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008977
8978 video_stream_encoder_->Stop();
8979 }
8980
8981 void ConfigureEncoder(const VideoStream& stream) {
8982 VideoEncoderConfig config;
Jonas Oreland80c87d72022-09-29 15:01:09 +02008983 webrtc::VideoEncoder::EncoderInfo encoder_info;
8984
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008985 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8986 config.max_bitrate_bps = stream.max_bitrate_bps;
8987 config.simulcast_layers[0] = stream;
8988 config.video_stream_factory =
8989 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8990 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
Jonas Oreland80c87d72022-09-29 15:01:09 +02008991 /*conference_mode=*/false, encoder_info);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008992 video_stream_encoder_->ConfigureEncoder(std::move(config),
8993 kMaxPayloadLength);
8994 }
8995
8996 void OnBitrateUpdated(DataRate bitrate) {
8997 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8998 bitrate, bitrate, bitrate, 0, 0, 0);
8999 }
9000
9001 void InsertFrameAndWaitForEncoded() {
9002 timestamp_ms_ += kFrameIntervalMs;
9003 video_source_.IncomingCapturedFrame(
9004 CreateFrame(timestamp_ms_, kWidth, kHeight));
9005 sink_.WaitForEncodedFrame(timestamp_ms_);
9006 }
9007
9008 void ExpectEqual(const VideoCodec& actual,
9009 const VideoStream& expected) const {
9010 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
9011 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
9012 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
9013 static_cast<unsigned int>(expected.min_bitrate_bps));
9014 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
9015 static_cast<unsigned int>(expected.max_bitrate_bps));
9016 EXPECT_EQ(actual.simulcastStream[0].width,
9017 kWidth / expected.scale_resolution_down_by);
9018 EXPECT_EQ(actual.simulcastStream[0].height,
9019 kHeight / expected.scale_resolution_down_by);
9020 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
9021 expected.num_temporal_layers);
Niels Möller79d566b2022-04-29 11:03:13 +02009022 EXPECT_EQ(actual.GetScalabilityMode(), expected.scalability_mode);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02009023 }
9024
9025 VideoStream DefaultConfig() const {
9026 VideoStream stream;
9027 stream.max_framerate = 25;
9028 stream.min_bitrate_bps = 35000;
9029 stream.max_bitrate_bps = 900000;
9030 stream.scale_resolution_down_by = 1.0;
9031 stream.num_temporal_layers = 1;
9032 stream.bitrate_priority = 1.0;
Niels Möller79d566b2022-04-29 11:03:13 +02009033 stream.scalability_mode = absl::nullopt;
Åsa Persson4d4f62f2021-09-12 16:14:48 +02009034 return stream;
9035 }
9036
9037 const int kWidth = 640;
9038 const int kHeight = 360;
9039 int64_t timestamp_ms_ = 0;
9040};
9041
9042TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
9043 VideoStream config1 = DefaultConfig();
9044 VideoStream config2 = config1;
9045 config2.max_framerate++;
9046
9047 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9048}
9049
9050TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
9051 VideoStream config1 = DefaultConfig();
9052 VideoStream config2 = config1;
9053 config2.min_bitrate_bps += 10000;
9054
9055 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9056}
9057
9058TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
9059 VideoStream config1 = DefaultConfig();
9060 VideoStream config2 = config1;
9061 config2.max_bitrate_bps += 100000;
9062
9063 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9064}
9065
9066TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
9067 VideoStream config1 = DefaultConfig();
9068 VideoStream config2 = config1;
9069 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
9070
9071 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9072}
9073
9074TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
9075 VideoStream config1 = DefaultConfig();
9076 VideoStream config2 = config1;
9077 config2.scale_resolution_down_by *= 2;
9078
9079 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9080}
9081
9082TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
9083 VideoStream config1 = DefaultConfig();
9084 VideoStream config2 = config1;
9085 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
9086
9087 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9088}
9089
9090TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
9091 VideoStream config1 = DefaultConfig();
9092 VideoStream config2 = config1;
Niels Möller05954892022-05-13 13:34:37 +02009093 config2.scalability_mode = ScalabilityMode::kL2T1;
9094
9095 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9096}
9097
9098TEST_F(ReconfigureEncoderTest,
9099 UpdatesNumTemporalLayersFromScalabilityModeChanges) {
9100 VideoStream config1 = DefaultConfig();
9101 VideoStream config2 = config1;
Niels Möller79d566b2022-04-29 11:03:13 +02009102 config2.scalability_mode = ScalabilityMode::kL1T2;
Niels Möller05954892022-05-13 13:34:37 +02009103 config2.num_temporal_layers = 2;
Åsa Persson4d4f62f2021-09-12 16:14:48 +02009104
9105 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9106}
9107
Tommi62b01db2022-01-25 23:41:22 +01009108// Simple test that just creates and then immediately destroys an encoder.
9109// The purpose of the test is to make sure that nothing bad happens if the
9110// initialization step on the encoder queue, doesn't run.
9111TEST(VideoStreamEncoderSimpleTest, CreateDestroy) {
9112 class SuperLazyTaskQueue : public webrtc::TaskQueueBase {
9113 public:
9114 SuperLazyTaskQueue() = default;
9115 ~SuperLazyTaskQueue() override = default;
9116
9117 private:
9118 void Delete() override { delete this; }
Danil Chapovalov95eeaa72022-07-06 10:14:29 +02009119 void PostTask(absl::AnyInvocable<void() &&> task) override {
Tommi62b01db2022-01-25 23:41:22 +01009120 // meh.
9121 }
Danil Chapovalov95eeaa72022-07-06 10:14:29 +02009122 void PostDelayedTask(absl::AnyInvocable<void() &&> task,
9123 TimeDelta delay) override {
Tommi62b01db2022-01-25 23:41:22 +01009124 ASSERT_TRUE(false);
9125 }
Danil Chapovalov95eeaa72022-07-06 10:14:29 +02009126 void PostDelayedHighPrecisionTask(absl::AnyInvocable<void() &&> task,
9127 TimeDelta delay) override {
9128 ADD_FAILURE();
9129 }
Tommi62b01db2022-01-25 23:41:22 +01009130 };
9131
9132 // Lots of boiler plate.
Jonas Oreland8ca06132022-03-14 12:52:48 +01009133 test::ScopedKeyValueConfig field_trials;
Philipp Hanckea204ad22022-07-08 18:43:25 +02009134 GlobalSimulatedTimeController time_controller(Timestamp::Zero());
Tommi62b01db2022-01-25 23:41:22 +01009135 auto stats_proxy = std::make_unique<MockableSendStatisticsProxy>(
9136 time_controller.GetClock(), VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 12:52:48 +01009137 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo, field_trials);
Tommi62b01db2022-01-25 23:41:22 +01009138 SimpleVideoStreamEncoderFactory::MockFakeEncoder mock_fake_encoder(
9139 time_controller.GetClock());
9140 test::VideoEncoderProxyFactory encoder_factory(&mock_fake_encoder);
9141 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory =
9142 CreateBuiltinVideoBitrateAllocatorFactory();
9143 VideoStreamEncoderSettings encoder_settings{
9144 VideoEncoder::Capabilities(/*loss_notification=*/false)};
9145 encoder_settings.encoder_factory = &encoder_factory;
9146 encoder_settings.bitrate_allocator_factory = bitrate_allocator_factory.get();
9147
9148 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9149 EXPECT_CALL((*adapter.get()), Initialize).WillOnce(Return());
9150
9151 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
9152 encoder_queue(new SuperLazyTaskQueue());
9153
9154 // Construct a VideoStreamEncoder instance and let it go out of scope without
9155 // doing anything else (including calling Stop()). This should be fine since
9156 // the posted init task will simply be deleted.
9157 auto encoder = std::make_unique<VideoStreamEncoder>(
9158 time_controller.GetClock(), 1, stats_proxy.get(), encoder_settings,
Markus Handell8e4197b2022-05-30 15:45:28 +02009159 std::make_unique<CpuOveruseDetectorProxy>(stats_proxy.get(),
9160 field_trials),
Tommi62b01db2022-01-25 23:41:22 +01009161 std::move(adapter), std::move(encoder_queue),
9162 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009163 kVideoBitrateAllocation,
9164 field_trials);
Tommi4d6dd172022-05-23 09:34:41 +02009165
9166 // Stop the encoder explicitly. This additional step tests if we could
9167 // hang when calling stop and the TQ has been stopped and/or isn't accepting
9168 // any more tasks.
9169 encoder->Stop();
Tommi62b01db2022-01-25 23:41:22 +01009170}
9171
Markus Handellb4e96d42021-11-05 12:00:55 +01009172TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
9173 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9174 auto* adapter_ptr = adapter.get();
9175 SimpleVideoStreamEncoderFactory factory;
Markus Handell8d87c462021-12-16 11:37:16 +01009176 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9177 nullptr;
9178 EXPECT_CALL(*adapter_ptr, Initialize)
9179 .WillOnce(Invoke([&video_stream_encoder_callback](
9180 FrameCadenceAdapterInterface::Callback* callback) {
9181 video_stream_encoder_callback = callback;
9182 }));
9183 TaskQueueBase* encoder_queue = nullptr;
9184 auto video_stream_encoder =
9185 factory.Create(std::move(adapter), &encoder_queue);
Markus Handellb4e96d42021-11-05 12:00:55 +01009186
Markus Handelle59fee82021-12-23 09:29:23 +01009187 // First a call before we know the frame size and hence cannot compute the
9188 // number of simulcast layers.
9189 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
9190 &FrameCadenceAdapterInterface::
9191 ZeroHertzModeParams::num_simulcast_layers,
Markus Handell5a77e512022-09-01 12:51:50 +00009192 Eq(0u)))));
Markus Handellb4e96d42021-11-05 12:00:55 +01009193 VideoEncoderConfig config;
Markus Handell8d87c462021-12-16 11:37:16 +01009194 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
Markus Handellb4e96d42021-11-05 12:00:55 +01009195 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9196 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
Markus Handelle59fee82021-12-23 09:29:23 +01009197 factory.DepleteTaskQueues();
9198
9199 // Then a call as we've computed the number of simulcast layers after a passed
9200 // frame.
9201 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
9202 &FrameCadenceAdapterInterface::
9203 ZeroHertzModeParams::num_simulcast_layers,
Markus Handell5a77e512022-09-01 12:51:50 +00009204 Gt(0u)))));
Markus Handell8d87c462021-12-16 11:37:16 +01009205 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handell9a478b52021-11-18 16:07:01 +01009206 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01009207 Mock::VerifyAndClearExpectations(adapter_ptr);
9208
Markus Handelle59fee82021-12-23 09:29:23 +01009209 // Expect a disabled zero-hertz mode after passing realtime video.
Markus Handell8d87c462021-12-16 11:37:16 +01009210 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Eq(absl::nullopt)));
Markus Handellb4e96d42021-11-05 12:00:55 +01009211 VideoEncoderConfig config2;
Markus Handell8d87c462021-12-16 11:37:16 +01009212 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config2);
Markus Handellb4e96d42021-11-05 12:00:55 +01009213 config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
9214 video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
Markus Handell8d87c462021-12-16 11:37:16 +01009215 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
Markus Handell9a478b52021-11-18 16:07:01 +01009216 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01009217}
9218
9219TEST(VideoStreamEncoderFrameCadenceTest,
9220 ForwardsFramesIntoFrameCadenceAdapter) {
9221 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9222 auto* adapter_ptr = adapter.get();
9223 test::FrameForwarder video_source;
9224 SimpleVideoStreamEncoderFactory factory;
9225 auto video_stream_encoder = factory.Create(std::move(adapter));
9226 video_stream_encoder->SetSource(
9227 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9228
9229 EXPECT_CALL(*adapter_ptr, OnFrame);
9230 auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
9231 video_source.IncomingCapturedFrame(
Markus Handell8d87c462021-12-16 11:37:16 +01009232 VideoFrame::Builder().set_video_frame_buffer(std::move(buffer)).build());
Markus Handellb4e96d42021-11-05 12:00:55 +01009233}
9234
Markus Handellee225432021-11-29 12:35:12 +01009235TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) {
9236 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9237 auto* adapter_ptr = adapter.get();
9238 test::FrameForwarder video_source;
9239 SimpleVideoStreamEncoderFactory factory;
9240 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9241 nullptr;
9242 EXPECT_CALL(*adapter_ptr, Initialize)
9243 .WillOnce(Invoke([&video_stream_encoder_callback](
9244 FrameCadenceAdapterInterface::Callback* callback) {
9245 video_stream_encoder_callback = callback;
9246 }));
9247 TaskQueueBase* encoder_queue = nullptr;
9248 auto video_stream_encoder =
9249 factory.Create(std::move(adapter), &encoder_queue);
9250
9251 // This is just to make the VSE operational. We'll feed a frame directly by
9252 // the callback interface.
9253 video_stream_encoder->SetSource(
9254 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9255
9256 VideoEncoderConfig video_encoder_config;
9257 test::FillEncoderConfiguration(kVideoCodecGeneric, 1, &video_encoder_config);
9258 video_stream_encoder->ConfigureEncoder(std::move(video_encoder_config),
9259 /*max_data_payload_length=*/1000);
9260
9261 EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps);
9262 EXPECT_CALL(*adapter_ptr, UpdateFrameRate);
Markus Handell8d87c462021-12-16 11:37:16 +01009263 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handellee225432021-11-29 12:35:12 +01009264 factory.DepleteTaskQueues();
9265}
9266
Markus Handell8d87c462021-12-16 11:37:16 +01009267TEST(VideoStreamEncoderFrameCadenceTest,
9268 DeactivatesActivatesLayersOnBitrateChanges) {
9269 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9270 auto* adapter_ptr = adapter.get();
9271 SimpleVideoStreamEncoderFactory factory;
9272 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9273 nullptr;
9274 EXPECT_CALL(*adapter_ptr, Initialize)
9275 .WillOnce(Invoke([&video_stream_encoder_callback](
9276 FrameCadenceAdapterInterface::Callback* callback) {
9277 video_stream_encoder_callback = callback;
9278 }));
9279 TaskQueueBase* encoder_queue = nullptr;
9280 auto video_stream_encoder =
9281 factory.Create(std::move(adapter), &encoder_queue);
9282
9283 // Configure 2 simulcast layers. FillEncoderConfiguration sets min bitrates to
9284 // {150000, 450000}.
9285 VideoEncoderConfig video_encoder_config;
9286 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9287 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9288 kMaxPayloadLength);
9289 // Ensure an encoder is created.
9290 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9291
9292 // Both layers enabled at 1 MBit/s.
9293 video_stream_encoder->OnBitrateUpdated(
9294 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9295 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9296 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9297 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9298 factory.DepleteTaskQueues();
9299 Mock::VerifyAndClearExpectations(adapter_ptr);
9300
9301 // Layer 1 disabled at 200 KBit/s.
9302 video_stream_encoder->OnBitrateUpdated(
9303 DataRate::KilobitsPerSec(200), DataRate::KilobitsPerSec(200),
9304 DataRate::KilobitsPerSec(200), 0, 0, 0);
9305 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9306 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9307 factory.DepleteTaskQueues();
9308 Mock::VerifyAndClearExpectations(adapter_ptr);
9309
9310 // All layers off at suspended video.
9311 video_stream_encoder->OnBitrateUpdated(DataRate::Zero(), DataRate::Zero(),
9312 DataRate::Zero(), 0, 0, 0);
9313 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/false));
9314 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9315 factory.DepleteTaskQueues();
9316 Mock::VerifyAndClearExpectations(adapter_ptr);
9317
9318 // Both layers enabled again back at 1 MBit/s.
9319 video_stream_encoder->OnBitrateUpdated(
9320 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9321 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9322 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9323 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9324 factory.DepleteTaskQueues();
9325}
9326
9327TEST(VideoStreamEncoderFrameCadenceTest, UpdatesQualityConvergence) {
9328 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9329 auto* adapter_ptr = adapter.get();
9330 SimpleVideoStreamEncoderFactory factory;
9331 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9332 nullptr;
9333 EXPECT_CALL(*adapter_ptr, Initialize)
9334 .WillOnce(Invoke([&video_stream_encoder_callback](
9335 FrameCadenceAdapterInterface::Callback* callback) {
9336 video_stream_encoder_callback = callback;
9337 }));
9338 TaskQueueBase* encoder_queue = nullptr;
9339 auto video_stream_encoder =
9340 factory.Create(std::move(adapter), &encoder_queue);
9341
9342 // Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder.
9343 VideoEncoderConfig video_encoder_config;
9344 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9345 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9346 kMaxPayloadLength);
9347 video_stream_encoder->OnBitrateUpdated(
9348 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9349 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9350
9351 // Pass a frame which has unconverged results.
9352 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9353 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9354 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9355 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
Markus Handelle1a198b2022-09-15 08:13:25 +00009356 encoded_image.qp_ = kVp8SteadyStateQpThreshold + 1;
Markus Handell8d87c462021-12-16 11:37:16 +01009357 CodecSpecificInfo codec_specific;
Markus Handelle1a198b2022-09-15 08:13:25 +00009358 codec_specific.codecType = kVideoCodecVP8;
Markus Handell8d87c462021-12-16 11:37:16 +01009359 return codec_specific;
9360 }));
9361 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, false));
9362 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9363 factory.DepleteTaskQueues();
9364 Mock::VerifyAndClearExpectations(adapter_ptr);
9365 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9366
9367 // Pass a frame which converges in layer 0 and not in layer 1.
9368 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9369 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9370 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9371 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
Markus Handelle1a198b2022-09-15 08:13:25 +00009372 // This sets spatial index 0 content to be at target quality, while
9373 // index 1 content is not.
9374 encoded_image.qp_ = kVp8SteadyStateQpThreshold +
9375 (encoded_image.SpatialIndex() == 0 ? 0 : 1);
Markus Handell8d87c462021-12-16 11:37:16 +01009376 CodecSpecificInfo codec_specific;
Markus Handelle1a198b2022-09-15 08:13:25 +00009377 codec_specific.codecType = kVideoCodecVP8;
Markus Handell8d87c462021-12-16 11:37:16 +01009378 return codec_specific;
9379 }));
9380 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, true));
9381 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9382 factory.DepleteTaskQueues();
9383 Mock::VerifyAndClearExpectations(adapter_ptr);
9384 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9385}
9386
Markus Handell2e0f4f02021-12-21 19:14:58 +01009387TEST(VideoStreamEncoderFrameCadenceTest,
9388 RequestsRefreshFramesWhenCadenceAdapterInstructs) {
9389 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9390 auto* adapter_ptr = adapter.get();
9391 MockVideoSourceInterface mock_source;
9392 SimpleVideoStreamEncoderFactory factory;
9393 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9394 nullptr;
9395 EXPECT_CALL(*adapter_ptr, Initialize)
9396 .WillOnce(Invoke([&video_stream_encoder_callback](
9397 FrameCadenceAdapterInterface::Callback* callback) {
9398 video_stream_encoder_callback = callback;
9399 }));
9400 TaskQueueBase* encoder_queue = nullptr;
9401 auto video_stream_encoder =
9402 factory.Create(std::move(adapter), &encoder_queue);
9403 video_stream_encoder->SetSource(
9404 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9405 VideoEncoderConfig config;
9406 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9407 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9408 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9409 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9410 // Ensure the encoder is set up.
9411 factory.DepleteTaskQueues();
9412
Markus Handell818e7fb2021-12-30 13:01:33 +01009413 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest)
9414 .WillOnce(Invoke([video_stream_encoder_callback] {
9415 video_stream_encoder_callback->RequestRefreshFrame();
9416 }));
Markus Handell2e0f4f02021-12-21 19:14:58 +01009417 EXPECT_CALL(mock_source, RequestRefreshFrame);
9418 video_stream_encoder->SendKeyFrame();
9419 factory.DepleteTaskQueues();
9420 Mock::VerifyAndClearExpectations(adapter_ptr);
9421 Mock::VerifyAndClearExpectations(&mock_source);
9422
Markus Handell818e7fb2021-12-30 13:01:33 +01009423 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest);
Markus Handell2e0f4f02021-12-21 19:14:58 +01009424 EXPECT_CALL(mock_source, RequestRefreshFrame).Times(0);
9425 video_stream_encoder->SendKeyFrame();
9426 factory.DepleteTaskQueues();
9427}
9428
Markus Handell818e7fb2021-12-30 13:01:33 +01009429TEST(VideoStreamEncoderFrameCadenceTest,
9430 RequestsRefreshFrameForEarlyZeroHertzKeyFrameRequest) {
9431 SimpleVideoStreamEncoderFactory factory;
9432 auto encoder_queue =
9433 factory.GetTimeController()->GetTaskQueueFactory()->CreateTaskQueue(
9434 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
9435
9436 // Enables zero-hertz mode.
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009437 test::ScopedKeyValueConfig field_trials(
9438 "WebRTC-ZeroHertzScreenshare/Enabled/");
Markus Handell818e7fb2021-12-30 13:01:33 +01009439 auto adapter = FrameCadenceAdapterInterface::Create(
Jonas Oreland8ca06132022-03-14 12:52:48 +01009440 factory.GetTimeController()->GetClock(), encoder_queue.get(),
9441 field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009442 FrameCadenceAdapterInterface* adapter_ptr = adapter.get();
9443
9444 MockVideoSourceInterface mock_source;
9445 auto video_stream_encoder = factory.CreateWithEncoderQueue(
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009446 std::move(adapter), std::move(encoder_queue), &field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009447
9448 video_stream_encoder->SetSource(
9449 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9450 VideoEncoderConfig config;
9451 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9452 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9453 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9454
9455 // Eventually expect a refresh frame request when requesting a key frame
9456 // before initializing zero-hertz mode. This can happen in reality because the
9457 // threads invoking key frame requests and constraints setup aren't
9458 // synchronized.
9459 EXPECT_CALL(mock_source, RequestRefreshFrame);
9460 video_stream_encoder->SendKeyFrame();
Markus Handellf5a50792022-06-16 14:25:15 +02009461 constexpr int kMaxFps = 30;
9462 adapter_ptr->OnConstraintsChanged(VideoTrackSourceConstraints{0, kMaxFps});
9463 factory.GetTimeController()->AdvanceTime(
9464 TimeDelta::Seconds(1) *
9465 FrameCadenceAdapterInterface::kOnDiscardedFrameRefreshFramePeriod /
9466 kMaxFps);
Markus Handell818e7fb2021-12-30 13:01:33 +01009467}
9468
perkj26091b12016-09-01 01:17:40 -07009469} // namespace webrtc