blob: c5ff292a001f3826e9d13bff72e6a75ffa3f3f7a [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Markus Handell9a478b52021-11-18 16:07:01 +010020#include "api/rtp_parameters.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020021#include "api/task_queue/default_task_queue_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010022#include "api/task_queue/task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020023#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010024#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010025#include "api/test/mock_video_encoder_factory.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080026#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020027#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020028#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020029#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010030#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010031#include "api/video_codecs/sdp_video_format.h"
Elad Alon370f93a2019-06-11 14:57:57 +020032#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020033#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010034#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020035#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010036#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020037#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070038#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080039#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020040#include "media/engine/webrtc_video_engine.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010041#include "modules/video_coding/codecs/av1/libaom_av1_encoder.h"
42#include "modules/video_coding/codecs/h264/include/h264.h"
43#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
44#include "modules/video_coding/codecs/vp8/include/vp8.h"
45#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020046#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020047#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020048#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010049#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020050#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010051#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020052#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020053#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080054#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020055#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010056#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020057#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020058#include "test/encoder_settings.h"
59#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020060#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010061#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020062#include "test/gmock.h"
63#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010064#include "test/mappable_native_buffer.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020065#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020066#include "test/video_encoder_proxy_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010067#include "video/frame_cadence_adapter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020068#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070069
70namespace webrtc {
71
sprang57c2fff2017-01-16 06:24:02 -080072using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020073using ::testing::AllOf;
Per Kjellanderd0a8f512020-10-07 11:28:41 +020074using ::testing::AtLeast;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020075using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020076using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020077using ::testing::Ge;
78using ::testing::Gt;
Markus Handellee225432021-11-29 12:35:12 +010079using ::testing::Invoke;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020080using ::testing::Le;
81using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010082using ::testing::Matcher;
Markus Handellb4e96d42021-11-05 12:00:55 +010083using ::testing::Mock;
philipel9b058032020-02-10 11:30:00 +010084using ::testing::NiceMock;
Markus Handellb4e96d42021-11-05 12:00:55 +010085using ::testing::Optional;
philipel9b058032020-02-10 11:30:00 +010086using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010087using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020088using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080089
perkj803d97f2016-11-01 11:45:46 -070090namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020091const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010092const int kQpLow = 1;
93const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020094const int kMinFramerateFps = 2;
95const int kMinBalancedFramerateFps = 7;
96const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080097const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 10:07:11 +020098const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
99const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
100const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
101const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 07:02:22 -0800102const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -0700103const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200104const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +0200105const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200106const VideoEncoder::ResolutionBitrateLimits
107 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
108const VideoEncoder::ResolutionBitrateLimits
109 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800110
Asa Persson606d3cb2021-10-04 10:07:11 +0200111uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200112 0x00, 0x00, 0x03, 0x03, 0xF4,
113 0x05, 0x03, 0xC7, 0xE0, 0x1B,
114 0x41, 0x10, 0x8D, 0x00};
115
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100116const uint8_t kCodedFrameVp8Qp25[] = {
117 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
118 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
119 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
120
perkj803d97f2016-11-01 11:45:46 -0700121class TestBuffer : public webrtc::I420Buffer {
122 public:
123 TestBuffer(rtc::Event* event, int width, int height)
124 : I420Buffer(width, height), event_(event) {}
125
126 private:
127 friend class rtc::RefCountedObject<TestBuffer>;
128 ~TestBuffer() override {
129 if (event_)
130 event_->Set();
131 }
132 rtc::Event* const event_;
133};
134
Henrik Boström56db9ff2021-03-24 09:06:45 +0100135// A fake native buffer that can't be converted to I420. Upon scaling, it
136// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700137class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
138 public:
139 FakeNativeBuffer(rtc::Event* event, int width, int height)
140 : event_(event), width_(width), height_(height) {}
141 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
142 int width() const override { return width_; }
143 int height() const override { return height_; }
144 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
145 return nullptr;
146 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100147 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
148 int offset_x,
149 int offset_y,
150 int crop_width,
151 int crop_height,
152 int scaled_width,
153 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200154 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
155 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100156 }
Noah Richards51db4212019-06-12 06:59:12 -0700157
158 private:
159 friend class rtc::RefCountedObject<FakeNativeBuffer>;
160 ~FakeNativeBuffer() override {
161 if (event_)
162 event_->Set();
163 }
164 rtc::Event* const event_;
165 const int width_;
166 const int height_;
167};
168
Evan Shrubsole895556e2020-10-05 09:15:13 +0200169// A fake native buffer that is backed by an NV12 buffer.
170class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
171 public:
172 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
173 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
174
175 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
176 int width() const override { return nv12_buffer_->width(); }
177 int height() const override { return nv12_buffer_->height(); }
178 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
179 return nv12_buffer_->ToI420();
180 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200181 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
182 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
183 if (absl::c_find(types, Type::kNV12) != types.end()) {
184 return nv12_buffer_;
185 }
186 return nullptr;
187 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200188 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
189
190 private:
191 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
192 ~FakeNV12NativeBuffer() override {
193 if (event_)
194 event_->Set();
195 }
196 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
197 rtc::Event* const event_;
198};
199
Niels Möller7dc26b72017-12-06 10:27:48 +0100200class CpuOveruseDetectorProxy : public OveruseFrameDetector {
201 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200202 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
203 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200204 last_target_framerate_fps_(-1),
205 framerate_updated_event_(true /* manual_reset */,
206 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100207 virtual ~CpuOveruseDetectorProxy() {}
208
209 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200210 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100211 last_target_framerate_fps_ = framerate_fps;
212 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200213 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100214 }
215
216 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200217 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100218 return last_target_framerate_fps_;
219 }
220
Niels Möller4db138e2018-04-19 09:04:13 +0200221 CpuOveruseOptions GetOptions() { return options_; }
222
Henrik Boström381d1092020-05-12 18:49:07 +0200223 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
224
Niels Möller7dc26b72017-12-06 10:27:48 +0100225 private:
Markus Handella3765182020-07-08 13:13:32 +0200226 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100227 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200228 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100229};
230
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200231class FakeVideoSourceRestrictionsListener
232 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200233 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200234 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200235 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200236 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200237 RTC_DCHECK(was_restrictions_updated_);
238 }
239
240 rtc::Event* restrictions_updated_event() {
241 return &restrictions_updated_event_;
242 }
243
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200244 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200245 void OnVideoSourceRestrictionsUpdated(
246 VideoSourceRestrictions restrictions,
247 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200248 rtc::scoped_refptr<Resource> reason,
249 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200250 was_restrictions_updated_ = true;
251 restrictions_updated_event_.Set();
252 }
253
254 private:
255 bool was_restrictions_updated_;
256 rtc::Event restrictions_updated_event_;
257};
258
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200259auto WantsFps(Matcher<int> fps_matcher) {
260 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
261 fps_matcher);
262}
263
264auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
265 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
266 AllOf(max_pixel_matcher, Gt(0)));
267}
268
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200269auto ResolutionMax() {
270 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200271 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200272 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
273 Eq(absl::nullopt)));
274}
275
276auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200277 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200278}
279
280auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200281 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200282}
283
284auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200285 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200286}
287
288auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200289 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200290}
291
292auto FpsMaxResolutionMax() {
293 return AllOf(FpsMax(), ResolutionMax());
294}
295
296auto UnlimitedSinkWants() {
297 return AllOf(FpsUnlimited(), ResolutionMax());
298}
299
300auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
301 Matcher<int> fps_range_matcher;
302
303 if (last_frame_pixels <= 320 * 240) {
304 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200305 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200306 fps_range_matcher = AllOf(Ge(10), Le(15));
307 } else if (last_frame_pixels <= 640 * 480) {
308 fps_range_matcher = Ge(15);
309 } else {
310 fps_range_matcher = Eq(kDefaultFramerate);
311 }
312 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
313 fps_range_matcher);
314}
315
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200316auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
317 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
318 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
319}
320
321auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
322 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
323}
324
325auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
326 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
327}
328
329auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
330 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
331 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
332}
333
334auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
335 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
336 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
337}
338
339auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
340 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
341 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
342}
343
344auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
345 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
346 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
347}
348
mflodmancc3d4422017-08-03 08:27:51 -0700349class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700350 public:
Markus Handell9a478b52021-11-18 16:07:01 +0100351 VideoStreamEncoderUnderTest(
352 TimeController* time_controller,
353 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter,
354 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
355 encoder_queue,
356 SendStatisticsProxy* stats_proxy,
357 const VideoStreamEncoderSettings& settings,
358 VideoStreamEncoder::BitrateAllocationCallbackType
359 allocation_callback_type)
360 : VideoStreamEncoder(time_controller->GetClock(),
361 1 /* number_of_cores */,
362 stats_proxy,
363 settings,
364 std::unique_ptr<OveruseFrameDetector>(
365 overuse_detector_proxy_ =
366 new CpuOveruseDetectorProxy(stats_proxy)),
367 std::move(cadence_adapter),
368 std::move(encoder_queue),
369 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200370 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200371 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200372 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200373 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200374 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200375 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200376 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200377 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100378 }
perkj803d97f2016-11-01 11:45:46 -0700379
Henrik Boström381d1092020-05-12 18:49:07 +0200380 void SetSourceAndWaitForRestrictionsUpdated(
381 rtc::VideoSourceInterface<VideoFrame>* source,
382 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200383 FakeVideoSourceRestrictionsListener listener;
384 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200385 SetSource(source, degradation_preference);
386 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200387 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200388 }
389
390 void SetSourceAndWaitForFramerateUpdated(
391 rtc::VideoSourceInterface<VideoFrame>* source,
392 const DegradationPreference& degradation_preference) {
393 overuse_detector_proxy_->framerate_updated_event()->Reset();
394 SetSource(source, degradation_preference);
395 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
396 }
397
398 void OnBitrateUpdatedAndWaitForManagedResources(
399 DataRate target_bitrate,
400 DataRate stable_target_bitrate,
401 DataRate link_allocation,
402 uint8_t fraction_lost,
403 int64_t round_trip_time_ms,
404 double cwnd_reduce_ratio) {
405 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
406 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
407 // Bitrate is updated on the encoder queue.
408 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200409 }
410
kthelgason2fc52542017-03-03 00:24:41 -0800411 // This is used as a synchronisation mechanism, to make sure that the
412 // encoder queue is not blocked before we start sending it frames.
413 void WaitUntilTaskQueueIsIdle() {
Markus Handell28c71802021-11-08 10:11:55 +0100414 time_controller_->AdvanceTime(TimeDelta::Zero());
kthelgason2fc52542017-03-03 00:24:41 -0800415 }
416
Henrik Boström91aa7322020-04-28 12:24:33 +0200417 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200418 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200419 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200420 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200421 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200422 event.Set();
423 });
424 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100425 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200426 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200427
Henrik Boström91aa7322020-04-28 12:24:33 +0200428 void TriggerCpuUnderuse() {
429 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200430 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200431 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200432 event.Set();
433 });
434 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100435 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200436 }
kthelgason876222f2016-11-29 01:44:11 -0800437
Henrik Boström91aa7322020-04-28 12:24:33 +0200438 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200439 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200440 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200441 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200442 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200443 event.Set();
444 });
445 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100446 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200447 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200448 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200449 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200450 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200451 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200452 event.Set();
453 });
454 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100455 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200456 }
457
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200458 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100459 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200460 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
461 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200462 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700463};
464
Noah Richards51db4212019-06-12 06:59:12 -0700465// Simulates simulcast behavior and makes highest stream resolutions divisible
466// by 4.
467class CroppingVideoStreamFactory
468 : public VideoEncoderConfig::VideoStreamFactoryInterface {
469 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200470 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700471
472 private:
473 std::vector<VideoStream> CreateEncoderStreams(
474 int width,
475 int height,
476 const VideoEncoderConfig& encoder_config) override {
477 std::vector<VideoStream> streams = test::CreateVideoStreams(
478 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700479 return streams;
480 }
Noah Richards51db4212019-06-12 06:59:12 -0700481};
482
sprangb1ca0732017-02-01 08:38:12 -0800483class AdaptingFrameForwarder : public test::FrameForwarder {
484 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200485 explicit AdaptingFrameForwarder(TimeController* time_controller)
486 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700487 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800488
489 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200490 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800491 adaptation_enabled_ = enabled;
492 }
493
asaperssonfab67072017-04-04 05:51:49 -0700494 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200495 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800496 return adaptation_enabled_;
497 }
498
Henrik Boström1124ed12021-02-25 10:30:39 +0100499 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
500 // the resolution or frame rate was different than it is currently. If
501 // something else is modified, such as encoder resolutions, but the resolution
502 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700503 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200504 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700505 return last_wants_;
506 }
507
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200508 absl::optional<int> last_sent_width() const { return last_width_; }
509 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800510
sprangb1ca0732017-02-01 08:38:12 -0800511 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200512 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +0100513 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200514
sprangb1ca0732017-02-01 08:38:12 -0800515 int cropped_width = 0;
516 int cropped_height = 0;
517 int out_width = 0;
518 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700519 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000520 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
521 << "w=" << video_frame.width()
522 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700523 if (adapter_.AdaptFrameResolution(
524 video_frame.width(), video_frame.height(),
525 video_frame.timestamp_us() * 1000, &cropped_width,
526 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100527 VideoFrame adapted_frame =
528 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200529 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100530 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200531 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100532 .set_timestamp_ms(99)
533 .set_rotation(kVideoRotation_0)
534 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100535 if (video_frame.has_update_rect()) {
536 adapted_frame.set_update_rect(
537 video_frame.update_rect().ScaleWithFrame(
538 video_frame.width(), video_frame.height(), 0, 0,
539 video_frame.width(), video_frame.height(), out_width,
540 out_height));
541 }
sprangc5d62e22017-04-02 23:53:04 -0700542 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800543 last_width_.emplace(adapted_frame.width());
544 last_height_.emplace(adapted_frame.height());
545 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200546 last_width_ = absl::nullopt;
547 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700548 }
sprangb1ca0732017-02-01 08:38:12 -0800549 } else {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000550 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800551 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800552 last_width_.emplace(video_frame.width());
553 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800554 }
555 }
556
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200557 void OnOutputFormatRequest(int width, int height) {
558 absl::optional<std::pair<int, int>> target_aspect_ratio =
559 std::make_pair(width, height);
560 absl::optional<int> max_pixel_count = width * height;
561 absl::optional<int> max_fps;
562 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
563 max_fps);
564 }
565
sprangb1ca0732017-02-01 08:38:12 -0800566 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
567 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200568 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100569 rtc::VideoSinkWants prev_wants = sink_wants_locked();
570 bool did_adapt =
571 prev_wants.max_pixel_count != wants.max_pixel_count ||
572 prev_wants.target_pixel_count != wants.target_pixel_count ||
573 prev_wants.max_framerate_fps != wants.max_framerate_fps;
574 if (did_adapt) {
575 last_wants_ = prev_wants;
576 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100577 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200578 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800579 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200580
581 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800582 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200583 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
584 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200585 absl::optional<int> last_width_;
586 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800587};
sprangc5d62e22017-04-02 23:53:04 -0700588
Niels Möller213618e2018-07-24 09:29:58 +0200589// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700590class MockableSendStatisticsProxy : public SendStatisticsProxy {
591 public:
592 MockableSendStatisticsProxy(Clock* clock,
593 const VideoSendStream::Config& config,
594 VideoEncoderConfig::ContentType content_type)
595 : SendStatisticsProxy(clock, config, content_type) {}
596
597 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200598 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700599 if (mock_stats_)
600 return *mock_stats_;
601 return SendStatisticsProxy::GetStats();
602 }
603
Niels Möller213618e2018-07-24 09:29:58 +0200604 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200605 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200606 if (mock_stats_)
607 return mock_stats_->input_frame_rate;
608 return SendStatisticsProxy::GetInputFrameRate();
609 }
sprangc5d62e22017-04-02 23:53:04 -0700610 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200611 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700612 mock_stats_.emplace(stats);
613 }
614
615 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200616 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700617 mock_stats_.reset();
618 }
619
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200620 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
621 on_frame_dropped_ = std::move(callback);
622 }
623
sprangc5d62e22017-04-02 23:53:04 -0700624 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200625 void OnFrameDropped(DropReason reason) override {
626 SendStatisticsProxy::OnFrameDropped(reason);
627 if (on_frame_dropped_)
628 on_frame_dropped_(reason);
629 }
630
Markus Handella3765182020-07-08 13:13:32 +0200631 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200632 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200633 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700634};
635
Markus Handellb4e96d42021-11-05 12:00:55 +0100636class SimpleVideoStreamEncoderFactory {
637 public:
638 class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
639 public:
640 using VideoStreamEncoder::VideoStreamEncoder;
641 ~AdaptedVideoStreamEncoder() { Stop(); }
642 };
643
Markus Handellee225432021-11-29 12:35:12 +0100644 SimpleVideoStreamEncoderFactory() {
Markus Handellb4e96d42021-11-05 12:00:55 +0100645 encoder_settings_.encoder_factory = &encoder_factory_;
Markus Handellee225432021-11-29 12:35:12 +0100646 encoder_settings_.bitrate_allocator_factory =
647 bitrate_allocator_factory_.get();
Markus Handellb4e96d42021-11-05 12:00:55 +0100648 }
649
650 std::unique_ptr<AdaptedVideoStreamEncoder> Create(
Markus Handellee225432021-11-29 12:35:12 +0100651 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
652 TaskQueueBase** encoder_queue_ptr = nullptr) {
653 auto encoder_queue =
654 time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
655 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
656 if (encoder_queue_ptr)
657 *encoder_queue_ptr = encoder_queue.get();
Markus Handellb4e96d42021-11-05 12:00:55 +0100658 auto result = std::make_unique<AdaptedVideoStreamEncoder>(
659 time_controller_.GetClock(),
660 /*number_of_cores=*/1,
661 /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
662 std::make_unique<CpuOveruseDetectorProxy>(/*stats_proxy=*/nullptr),
Markus Handellee225432021-11-29 12:35:12 +0100663 std::move(zero_hertz_adapter), std::move(encoder_queue),
Markus Handellb4e96d42021-11-05 12:00:55 +0100664 VideoStreamEncoder::BitrateAllocationCallbackType::
665 kVideoBitrateAllocation);
666 result->SetSink(&sink_, /*rotation_applied=*/false);
667 return result;
668 }
669
Markus Handell9a478b52021-11-18 16:07:01 +0100670 void DepleteTaskQueues() { time_controller_.AdvanceTime(TimeDelta::Zero()); }
671
Markus Handellb4e96d42021-11-05 12:00:55 +0100672 private:
673 class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
674 public:
675 ~NullEncoderSink() override = default;
676 void OnEncoderConfigurationChanged(
677 std::vector<VideoStream> streams,
678 bool is_svc,
679 VideoEncoderConfig::ContentType content_type,
680 int min_transmit_bitrate_bps) override {}
681 void OnBitrateAllocationUpdated(
682 const VideoBitrateAllocation& allocation) override {}
683 void OnVideoLayersAllocationUpdated(
684 VideoLayersAllocation allocation) override {}
685 Result OnEncodedImage(
686 const EncodedImage& encoded_image,
687 const CodecSpecificInfo* codec_specific_info) override {
688 return Result(EncodedImageCallback::Result::OK);
689 }
690 };
691
Markus Handellee225432021-11-29 12:35:12 +0100692 GlobalSimulatedTimeController time_controller_{Timestamp::Millis(0)};
693 std::unique_ptr<TaskQueueFactory> task_queue_factory_{
694 time_controller_.CreateTaskQueueFactory()};
695 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_ =
696 std::make_unique<MockableSendStatisticsProxy>(
697 time_controller_.GetClock(),
698 VideoSendStream::Config(nullptr),
699 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo);
700 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_ =
701 CreateBuiltinVideoBitrateAllocatorFactory();
702 VideoStreamEncoderSettings encoder_settings_{
703 VideoEncoder::Capabilities(/*loss_notification=*/false)};
704 test::FakeEncoder fake_encoder_{time_controller_.GetClock()};
705 test::VideoEncoderProxyFactory encoder_factory_{&fake_encoder_};
Markus Handellb4e96d42021-11-05 12:00:55 +0100706 NullEncoderSink sink_;
707};
708
709class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
710 public:
711 MOCK_METHOD(void, Initialize, (Callback * callback), (override));
712 MOCK_METHOD(void, SetZeroHertzModeEnabled, (bool), (override));
713 MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
Markus Handellee225432021-11-29 12:35:12 +0100714 MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override));
715 MOCK_METHOD(void, UpdateFrameRate, (), (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100716};
717
philipel9b058032020-02-10 11:30:00 +0100718class MockEncoderSelector
719 : public VideoEncoderFactory::EncoderSelectorInterface {
720 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200721 MOCK_METHOD(void,
722 OnCurrentEncoder,
723 (const SdpVideoFormat& format),
724 (override));
725 MOCK_METHOD(absl::optional<SdpVideoFormat>,
726 OnAvailableBitrate,
727 (const DataRate& rate),
728 (override));
729 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100730};
731
perkj803d97f2016-11-01 11:45:46 -0700732} // namespace
733
mflodmancc3d4422017-08-03 08:27:51 -0700734class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700735 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200736 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700737
mflodmancc3d4422017-08-03 08:27:51 -0700738 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700739 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700740 codec_width_(320),
741 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200742 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200743 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200744 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700745 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200746 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700747 video_send_config_,
748 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200749 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700750
751 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700752 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700753 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200754 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800755 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200756 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200757 video_send_config_.rtp.payload_name = "FAKE";
758 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700759
Per512ecb32016-09-23 15:52:06 +0200760 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200761 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200762 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
763 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
764 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100765 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700766
Niels Möllerf1338562018-04-26 09:51:47 +0200767 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800768 }
769
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100770 void ConfigureEncoder(
771 VideoEncoderConfig video_encoder_config,
772 VideoStreamEncoder::BitrateAllocationCallbackType
773 allocation_callback_type =
774 VideoStreamEncoder::BitrateAllocationCallbackType::
775 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 08:27:51 -0700776 if (video_stream_encoder_)
777 video_stream_encoder_->Stop();
Markus Handell9a478b52021-11-18 16:07:01 +0100778
779 auto encoder_queue = GetTaskQueueFactory()->CreateTaskQueue(
780 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
781 TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
782 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter =
783 FrameCadenceAdapterInterface::Create(time_controller_.GetClock(),
784 encoder_queue_ptr);
785 video_stream_encoder_ = std::make_unique<VideoStreamEncoderUnderTest>(
786 &time_controller_, std::move(cadence_adapter), std::move(encoder_queue),
787 stats_proxy_.get(), video_send_config_.encoder_settings,
788 allocation_callback_type);
Asa Persson606d3cb2021-10-04 10:07:11 +0200789 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 08:27:51 -0700790 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700791 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 10:07:11 +0200792 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 08:27:51 -0700793 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200794 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700795 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800796 }
797
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100798 void ResetEncoder(const std::string& payload_name,
799 size_t num_streams,
800 size_t num_temporal_layers,
801 unsigned char num_spatial_layers,
802 bool screenshare,
803 VideoStreamEncoder::BitrateAllocationCallbackType
804 allocation_callback_type =
805 VideoStreamEncoder::BitrateAllocationCallbackType::
806 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200807 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800808
809 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200810 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
811 num_streams, &video_encoder_config);
812 for (auto& layer : video_encoder_config.simulcast_layers) {
813 layer.num_temporal_layers = num_temporal_layers;
814 layer.max_framerate = kDefaultFramerate;
815 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100816 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 10:07:11 +0200817 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 07:06:52 -0700818 video_encoder_config.content_type =
819 screenshare ? VideoEncoderConfig::ContentType::kScreen
820 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700821 if (payload_name == "VP9") {
822 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
823 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200824 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700825 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200826 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
827 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700828 }
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100829 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 01:17:40 -0700830 }
831
sprang57c2fff2017-01-16 06:24:02 -0800832 VideoFrame CreateFrame(int64_t ntp_time_ms,
833 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200834 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200835 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200836 destruction_event, codec_width_, codec_height_))
837 .set_ntp_time_ms(ntp_time_ms)
838 .set_timestamp_ms(99)
839 .set_rotation(kVideoRotation_0)
840 .build();
perkj26091b12016-09-01 01:17:40 -0700841 }
842
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100843 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
844 rtc::Event* destruction_event,
845 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200846 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200847 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200848 destruction_event, codec_width_, codec_height_))
849 .set_ntp_time_ms(ntp_time_ms)
850 .set_timestamp_ms(99)
851 .set_rotation(kVideoRotation_0)
852 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
853 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100854 }
855
sprang57c2fff2017-01-16 06:24:02 -0800856 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200857 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
858 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200859 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200860 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200861 .set_ntp_time_ms(ntp_time_ms)
862 .set_timestamp_ms(ntp_time_ms)
863 .set_rotation(kVideoRotation_0)
864 .build();
perkj803d97f2016-11-01 11:45:46 -0700865 }
866
Evan Shrubsole895556e2020-10-05 09:15:13 +0200867 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200868 return VideoFrame::Builder()
869 .set_video_frame_buffer(NV12Buffer::Create(width, height))
870 .set_ntp_time_ms(ntp_time_ms)
871 .set_timestamp_ms(ntp_time_ms)
872 .set_rotation(kVideoRotation_0)
873 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200874 }
875
Noah Richards51db4212019-06-12 06:59:12 -0700876 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
877 rtc::Event* destruction_event,
878 int width,
879 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200880 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200881 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200882 destruction_event, width, height))
883 .set_ntp_time_ms(ntp_time_ms)
884 .set_timestamp_ms(99)
885 .set_rotation(kVideoRotation_0)
886 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700887 }
888
Evan Shrubsole895556e2020-10-05 09:15:13 +0200889 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
890 rtc::Event* destruction_event,
891 int width,
892 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200893 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200894 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200895 destruction_event, width, height))
896 .set_ntp_time_ms(ntp_time_ms)
897 .set_timestamp_ms(99)
898 .set_rotation(kVideoRotation_0)
899 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200900 }
901
Noah Richards51db4212019-06-12 06:59:12 -0700902 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
903 rtc::Event* destruction_event) const {
904 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
905 codec_height_);
906 }
907
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100908 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200909 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +0200910 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100911
912 video_source_.IncomingCapturedFrame(
913 CreateFrame(1, codec_width_, codec_height_));
914 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200915 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100916 }
917
sprang4847ae62017-06-27 07:06:52 -0700918 void WaitForEncodedFrame(int64_t expected_ntp_time) {
919 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200920 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700921 }
922
923 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
924 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200925 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700926 return ok;
927 }
928
929 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
930 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200931 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700932 }
933
934 void ExpectDroppedFrame() {
935 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200936 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700937 }
938
939 bool WaitForFrame(int64_t timeout_ms) {
940 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200941 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700942 return ok;
943 }
944
perkj26091b12016-09-01 01:17:40 -0700945 class TestEncoder : public test::FakeEncoder {
946 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200947 explicit TestEncoder(TimeController* time_controller)
948 : FakeEncoder(time_controller->GetClock()),
949 time_controller_(time_controller) {
950 RTC_DCHECK(time_controller_);
951 }
perkj26091b12016-09-01 01:17:40 -0700952
Erik Språngaed30702018-11-05 12:57:17 +0100953 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +0200954 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +0200955 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +0100956 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100957 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +0100958 info.scaling_settings = VideoEncoder::ScalingSettings(
959 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +0100960 }
961 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100962 for (int i = 0; i < kMaxSpatialLayers; ++i) {
963 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100964 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100965 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +0100966 for (int tid = 0; tid < num_layers; ++tid)
967 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100968 }
969 }
Erik Språngaed30702018-11-05 12:57:17 +0100970 }
Sergey Silkin6456e352019-07-08 17:56:40 +0200971
972 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +0100973 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200974 info.apply_alignment_to_all_simulcast_layers =
975 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200976 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-30 06:48:03 +0800977 if (is_qp_trusted_.has_value()) {
978 info.is_qp_trusted = is_qp_trusted_;
979 }
Erik Språngaed30702018-11-05 12:57:17 +0100980 return info;
kthelgason876222f2016-11-29 01:44:11 -0800981 }
982
Erik Språngb7cb7b52019-02-26 15:52:33 +0100983 int32_t RegisterEncodeCompleteCallback(
984 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +0200985 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +0100986 encoded_image_callback_ = callback;
987 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
988 }
989
perkjfa10b552016-10-02 23:45:26 -0700990 void ContinueEncode() { continue_encode_event_.Set(); }
991
992 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
993 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +0200994 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -0700995 EXPECT_EQ(timestamp_, timestamp);
996 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
997 }
998
kthelgason2fc52542017-03-03 00:24:41 -0800999 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +02001000 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -08001001 quality_scaling_ = b;
1002 }
kthelgasonad9010c2017-02-14 00:46:51 -08001003
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001004 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +02001005 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001006 requested_resolution_alignment_ = requested_resolution_alignment;
1007 }
1008
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001009 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
1010 MutexLock lock(&local_mutex_);
1011 apply_alignment_to_all_simulcast_layers_ = b;
1012 }
1013
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001014 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +02001015 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001016 is_hardware_accelerated_ = is_hardware_accelerated;
1017 }
1018
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001019 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
1020 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +02001021 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001022 temporal_layers_supported_[spatial_idx] = supported;
1023 }
1024
Sergey Silkin6456e352019-07-08 17:56:40 +02001025 void SetResolutionBitrateLimits(
1026 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +02001027 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +02001028 resolution_bitrate_limits_ = thresholds;
1029 }
1030
sprangfe627f32017-03-29 08:24:59 -07001031 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +02001032 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -07001033 force_init_encode_failed_ = force_failure;
1034 }
1035
Niels Möller6bb5ab92019-01-11 11:11:10 +01001036 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +02001037 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001038 rate_factor_ = rate_factor;
1039 }
1040
Erik Språngd7329ca2019-02-21 21:19:53 +01001041 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +02001042 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001043 return last_framerate_;
1044 }
1045
Erik Språngd7329ca2019-02-21 21:19:53 +01001046 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +02001047 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001048 return last_update_rect_;
1049 }
1050
Niels Möller87e2d782019-03-07 10:18:23 +01001051 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +02001052 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001053 return last_frame_types_;
1054 }
1055
1056 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +01001057 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +01001058 keyframe ? VideoFrameType::kVideoFrameKey
1059 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01001060 {
Markus Handella3765182020-07-08 13:13:32 +02001061 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001062 last_frame_types_ = frame_type;
1063 }
Niels Möllerb859b322019-03-07 12:40:01 +01001064 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +01001065 }
1066
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001067 void InjectEncodedImage(const EncodedImage& image,
1068 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +02001069 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001070 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001071 }
1072
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001073 void SetEncodedImageData(
1074 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +02001075 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001076 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001077 }
1078
Erik Språngd7329ca2019-02-21 21:19:53 +01001079 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +02001080 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001081 expect_null_frame_ = true;
1082 }
1083
Erik Språng5056af02019-09-02 15:53:11 +02001084 absl::optional<VideoEncoder::RateControlParameters>
1085 GetAndResetLastRateControlSettings() {
1086 auto settings = last_rate_control_settings_;
1087 last_rate_control_settings_.reset();
1088 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +01001089 }
1090
Henrik Boström56db9ff2021-03-24 09:06:45 +01001091 int GetLastInputWidth() const {
1092 MutexLock lock(&local_mutex_);
1093 return last_input_width_;
1094 }
1095
1096 int GetLastInputHeight() const {
1097 MutexLock lock(&local_mutex_);
1098 return last_input_height_;
1099 }
1100
Evan Shrubsole895556e2020-10-05 09:15:13 +02001101 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1102 MutexLock lock(&local_mutex_);
1103 return last_input_pixel_format_;
1104 }
1105
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001106 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001107 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001108 return num_set_rates_;
1109 }
1110
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001111 void SetPreferredPixelFormats(
1112 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1113 pixel_formats) {
1114 MutexLock lock(&local_mutex_);
1115 preferred_pixel_formats_ = std::move(pixel_formats);
1116 }
1117
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001118 void SetIsQpTrusted(absl::optional<bool> trusted) {
1119 MutexLock lock(&local_mutex_);
1120 is_qp_trusted_ = trusted;
1121 }
1122
perkjfa10b552016-10-02 23:45:26 -07001123 private:
perkj26091b12016-09-01 01:17:40 -07001124 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001125 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001126 {
Markus Handella3765182020-07-08 13:13:32 +02001127 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001128 if (expect_null_frame_) {
1129 EXPECT_EQ(input_image.timestamp(), 0u);
1130 EXPECT_EQ(input_image.width(), 1);
1131 last_frame_types_ = *frame_types;
1132 expect_null_frame_ = false;
1133 } else {
1134 EXPECT_GT(input_image.timestamp(), timestamp_);
1135 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1136 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1137 }
perkj26091b12016-09-01 01:17:40 -07001138
1139 timestamp_ = input_image.timestamp();
1140 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001141 last_input_width_ = input_image.width();
1142 last_input_height_ = input_image.height();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001143 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001144 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001145 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001146 }
Niels Möllerb859b322019-03-07 12:40:01 +01001147 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001148 return result;
1149 }
1150
Niels Möller08ae7ce2020-09-23 15:58:12 +02001151 CodecSpecificInfo EncodeHook(
1152 EncodedImage& encoded_image,
1153 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001154 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001155 {
1156 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001157 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001158 }
1159 MutexLock lock(&local_mutex_);
1160 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001161 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001162 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001163 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001164 }
1165
sprangfe627f32017-03-29 08:24:59 -07001166 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001167 const Settings& settings) override {
1168 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001169
Markus Handella3765182020-07-08 13:13:32 +02001170 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001171 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001172
Erik Språng82fad3d2018-03-21 09:57:23 +01001173 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001174 // Simulate setting up temporal layers, in order to validate the life
1175 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001176 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001177 frame_buffer_controller_ =
1178 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001179 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001180 if (force_init_encode_failed_) {
1181 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001182 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001183 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001184
Erik Språngb7cb7b52019-02-26 15:52:33 +01001185 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001186 return res;
1187 }
1188
Erik Språngb7cb7b52019-02-26 15:52:33 +01001189 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001190 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001191 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1192 initialized_ = EncoderState::kUninitialized;
1193 return FakeEncoder::Release();
1194 }
1195
Erik Språng16cb8f52019-04-12 13:59:09 +02001196 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001197 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001198 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001199 VideoBitrateAllocation adjusted_rate_allocation;
1200 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1201 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001202 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001203 adjusted_rate_allocation.SetBitrate(
1204 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001205 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001206 rate_factor_));
1207 }
1208 }
1209 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001210 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001211 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001212 RateControlParameters adjusted_paramters = parameters;
1213 adjusted_paramters.bitrate = adjusted_rate_allocation;
1214 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001215 }
1216
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001217 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001218 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001219 enum class EncoderState {
1220 kUninitialized,
1221 kInitializationFailed,
1222 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001223 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
perkj26091b12016-09-01 01:17:40 -07001224 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001225 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1226 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1227 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1228 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1229 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1230 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001231 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1232 false;
Markus Handella3765182020-07-08 13:13:32 +02001233 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001234 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1235 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001236 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001237 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001238 absl::optional<bool>
1239 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001240 local_mutex_);
1241 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1242 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1243 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001244 absl::optional<VideoEncoder::RateControlParameters>
1245 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001246 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1247 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001248 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001249 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001250 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1251 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001252 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 17:56:40 +02001253 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001254 RTC_GUARDED_BY(local_mutex_);
1255 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001256 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1257 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001258 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1259 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001260 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001261 };
1262
mflodmancc3d4422017-08-03 08:27:51 -07001263 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001264 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001265 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1266 : time_controller_(time_controller), test_encoder_(test_encoder) {
1267 RTC_DCHECK(time_controller_);
1268 }
perkj26091b12016-09-01 01:17:40 -07001269
perkj26091b12016-09-01 01:17:40 -07001270 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001271 EXPECT_TRUE(
1272 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1273 }
1274
1275 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1276 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001277 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001278 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001279 return false;
perkj26091b12016-09-01 01:17:40 -07001280 {
Markus Handella3765182020-07-08 13:13:32 +02001281 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001282 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001283 }
1284 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001285 return true;
perkj26091b12016-09-01 01:17:40 -07001286 }
1287
sprangb1ca0732017-02-01 08:38:12 -08001288 void WaitForEncodedFrame(uint32_t expected_width,
1289 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001290 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001291 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001292 }
1293
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001294 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001295 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001296 uint32_t width = 0;
1297 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001298 {
Markus Handella3765182020-07-08 13:13:32 +02001299 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001300 width = last_width_;
1301 height = last_height_;
1302 }
1303 EXPECT_EQ(expected_height, height);
1304 EXPECT_EQ(expected_width, width);
1305 }
1306
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001307 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1308 VideoRotation rotation;
1309 {
Markus Handella3765182020-07-08 13:13:32 +02001310 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001311 rotation = last_rotation_;
1312 }
1313 EXPECT_EQ(expected_rotation, rotation);
1314 }
1315
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001316 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001317
sprangc5d62e22017-04-02 23:53:04 -07001318 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001319 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +01001320 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001321 bool ret = encoded_frame_event_.Wait(timeout_ms);
Markus Handell28c71802021-11-08 10:11:55 +01001322 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001323 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001324 }
1325
perkj26091b12016-09-01 01:17:40 -07001326 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001327 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001328 expect_frames_ = false;
1329 }
1330
asaperssonfab67072017-04-04 05:51:49 -07001331 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001332 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001333 return number_of_reconfigurations_;
1334 }
1335
asaperssonfab67072017-04-04 05:51:49 -07001336 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001337 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001338 return min_transmit_bitrate_bps_;
1339 }
1340
Erik Språngd7329ca2019-02-21 21:19:53 +01001341 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001342 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001343 num_expected_layers_ = num_layers;
1344 }
1345
Erik Språngb7cb7b52019-02-26 15:52:33 +01001346 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001347 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001348 return last_capture_time_ms_;
1349 }
1350
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001351 const EncodedImage& GetLastEncodedImage() {
1352 MutexLock lock(&mutex_);
1353 return last_encoded_image_;
1354 }
1355
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001356 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001357 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001358 return std::move(last_encoded_image_data_);
1359 }
1360
Per Kjellanderdcef6412020-10-07 15:09:05 +02001361 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1362 MutexLock lock(&mutex_);
1363 return last_bitrate_allocation_;
1364 }
1365
1366 int number_of_bitrate_allocations() const {
1367 MutexLock lock(&mutex_);
1368 return number_of_bitrate_allocations_;
1369 }
1370
Per Kjellandera9434842020-10-15 17:53:22 +02001371 VideoLayersAllocation GetLastVideoLayersAllocation() {
1372 MutexLock lock(&mutex_);
1373 return last_layers_allocation_;
1374 }
1375
1376 int number_of_layers_allocations() const {
1377 MutexLock lock(&mutex_);
1378 return number_of_layers_allocations_;
1379 }
1380
perkj26091b12016-09-01 01:17:40 -07001381 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001382 Result OnEncodedImage(
1383 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001384 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001385 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001386 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001387 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001388 last_encoded_image_data_ = std::vector<uint8_t>(
1389 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001390 uint32_t timestamp = encoded_image.Timestamp();
1391 if (last_timestamp_ != timestamp) {
1392 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001393 last_width_ = encoded_image._encodedWidth;
1394 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001395 } else {
1396 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001397 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1398 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001399 }
1400 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001401 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001402 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001403 if (num_received_layers_ == num_expected_layers_) {
1404 encoded_frame_event_.Set();
1405 }
sprangb1ca0732017-02-01 08:38:12 -08001406 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001407 }
1408
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001409 void OnEncoderConfigurationChanged(
1410 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001411 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001412 VideoEncoderConfig::ContentType content_type,
1413 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001414 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001415 ++number_of_reconfigurations_;
1416 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1417 }
1418
Per Kjellanderdcef6412020-10-07 15:09:05 +02001419 void OnBitrateAllocationUpdated(
1420 const VideoBitrateAllocation& allocation) override {
1421 MutexLock lock(&mutex_);
1422 ++number_of_bitrate_allocations_;
1423 last_bitrate_allocation_ = allocation;
1424 }
1425
Per Kjellandera9434842020-10-15 17:53:22 +02001426 void OnVideoLayersAllocationUpdated(
1427 VideoLayersAllocation allocation) override {
1428 MutexLock lock(&mutex_);
1429 ++number_of_layers_allocations_;
1430 last_layers_allocation_ = allocation;
1431 rtc::StringBuilder log;
1432 for (const auto& layer : allocation.active_spatial_layers) {
1433 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1434 << "[";
1435 for (const auto target_bitrate :
1436 layer.target_bitrate_per_temporal_layer) {
1437 log << target_bitrate.kbps() << ",";
1438 }
1439 log << "]";
1440 }
Harald Alvestrand97597c02021-11-04 12:01:23 +00001441 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 17:53:22 +02001442 }
1443
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001444 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001445 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001446 TestEncoder* test_encoder_;
1447 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001448 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001449 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001450 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001451 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001452 uint32_t last_height_ = 0;
1453 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001454 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001455 size_t num_expected_layers_ = 1;
1456 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001457 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001458 int number_of_reconfigurations_ = 0;
1459 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001460 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1461 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001462 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1463 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001464 };
1465
Sergey Silkin5ee69672019-07-02 14:18:34 +02001466 class VideoBitrateAllocatorProxyFactory
1467 : public VideoBitrateAllocatorFactory {
1468 public:
1469 VideoBitrateAllocatorProxyFactory()
1470 : bitrate_allocator_factory_(
1471 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1472
1473 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1474 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001475 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001476 codec_config_ = codec;
1477 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1478 }
1479
1480 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001481 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001482 return codec_config_;
1483 }
1484
1485 private:
1486 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1487
Markus Handella3765182020-07-08 13:13:32 +02001488 mutable Mutex mutex_;
1489 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001490 };
1491
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001492 Clock* clock() { return time_controller_.GetClock(); }
1493 void AdvanceTime(TimeDelta duration) {
1494 time_controller_.AdvanceTime(duration);
1495 }
1496
1497 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1498
1499 protected:
1500 virtual TaskQueueFactory* GetTaskQueueFactory() {
1501 return time_controller_.GetTaskQueueFactory();
1502 }
1503
1504 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001505 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001506 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001507 int codec_width_;
1508 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001509 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001510 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001511 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001512 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001513 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001514 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001515 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001516 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001517};
1518
mflodmancc3d4422017-08-03 08:27:51 -07001519TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001520 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001521 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001522 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001523 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001524 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001525 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001526 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001527}
1528
mflodmancc3d4422017-08-03 08:27:51 -07001529TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001530 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001531 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001532 // The encoder will cache up to one frame for a short duration. Adding two
1533 // frames means that the first frame will be dropped and the second frame will
1534 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001535 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001536 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001537 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Markus Handell28c71802021-11-08 10:11:55 +01001538 AdvanceTime(TimeDelta::Zero());
perkja49cbd32016-09-16 07:53:41 -07001539 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001540
Henrik Boström381d1092020-05-12 18:49:07 +02001541 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001542 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001543
Sebastian Janssona3177052018-04-10 13:05:49 +02001544 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001545 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001546 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1547
1548 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001549 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001550}
1551
mflodmancc3d4422017-08-03 08:27:51 -07001552TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001553 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001554 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001555 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001556 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001557
Henrik Boström381d1092020-05-12 18:49:07 +02001558 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001559 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1560
Sebastian Janssona3177052018-04-10 13:05:49 +02001561 // The encoder will cache up to one frame for a short duration. Adding two
1562 // frames means that the first frame will be dropped and the second frame will
1563 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001564 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001565 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001566
Henrik Boström381d1092020-05-12 18:49:07 +02001567 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001568 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001569 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001570 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1571 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001572 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001573}
1574
mflodmancc3d4422017-08-03 08:27:51 -07001575TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001576 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001577 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001578 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001579 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001580
1581 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001582 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001583
perkja49cbd32016-09-16 07:53:41 -07001584 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001585 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001586 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001587}
1588
mflodmancc3d4422017-08-03 08:27:51 -07001589TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001590 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001591 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001592
perkja49cbd32016-09-16 07:53:41 -07001593 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001594 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001595
mflodmancc3d4422017-08-03 08:27:51 -07001596 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001597 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001598 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001599 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1600 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001601}
1602
Markus Handell9a478b52021-11-18 16:07:01 +01001603TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
1604 test::FrameForwarder source;
1605 video_stream_encoder_->SetSource(&source,
1606 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02001607 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001608 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001609
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001610 int dropped_count = 0;
1611 stats_proxy_->SetDroppedFrameCallback(
1612 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1613 ++dropped_count;
1614 });
1615
Markus Handell9a478b52021-11-18 16:07:01 +01001616 source.IncomingCapturedFrame(CreateFrame(1, nullptr));
1617 source.IncomingCapturedFrame(CreateFrame(2, nullptr));
1618 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001619 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001620 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001621}
1622
Henrik Boström56db9ff2021-03-24 09:06:45 +01001623TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001624 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001625 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001626
1627 rtc::Event frame_destroyed_event;
1628 video_source_.IncomingCapturedFrame(
1629 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001630 WaitForEncodedFrame(1);
1631 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1632 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001633 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1634 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001635 video_stream_encoder_->Stop();
1636}
1637
Henrik Boström56db9ff2021-03-24 09:06:45 +01001638TEST_F(VideoStreamEncoderTest,
1639 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001640 // Use the cropping factory.
1641 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001642 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001643 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1644 kMaxPayloadLength);
1645 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1646
1647 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001648 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001649 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001650 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1651 WaitForEncodedFrame(1);
1652 // The encoder will have been configured once.
1653 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001654 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1655 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 06:59:12 -07001656
1657 // Now send in a fake frame that needs to be cropped as the width/height
1658 // aren't divisible by 4 (see CreateEncoderStreams above).
1659 rtc::Event frame_destroyed_event;
1660 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1661 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001662 WaitForEncodedFrame(2);
1663 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1664 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001665 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1666 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001667 video_stream_encoder_->Stop();
1668}
1669
Evan Shrubsole895556e2020-10-05 09:15:13 +02001670TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1671 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001672 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001673
1674 video_source_.IncomingCapturedFrame(
1675 CreateNV12Frame(1, codec_width_, codec_height_));
1676 WaitForEncodedFrame(1);
1677 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1678 fake_encoder_.GetLastInputPixelFormat());
1679 video_stream_encoder_->Stop();
1680}
1681
Henrik Boström56db9ff2021-03-24 09:06:45 +01001682TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001683 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001684 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001685
1686 fake_encoder_.SetPreferredPixelFormats({});
1687
1688 rtc::Event frame_destroyed_event;
1689 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1690 1, &frame_destroyed_event, codec_width_, codec_height_));
1691 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001692 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001693 fake_encoder_.GetLastInputPixelFormat());
1694 video_stream_encoder_->Stop();
1695}
1696
Henrik Boström56db9ff2021-03-24 09:06:45 +01001697TEST_F(VideoStreamEncoderTest,
1698 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001699 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001700 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001701
1702 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1703
1704 rtc::Event frame_destroyed_event;
1705 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1706 1, &frame_destroyed_event, codec_width_, codec_height_));
1707 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001708 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001709 fake_encoder_.GetLastInputPixelFormat());
1710 video_stream_encoder_->Stop();
1711}
1712
Henrik Boström56db9ff2021-03-24 09:06:45 +01001713TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001714 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001715 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001716
1717 // Fake NV12 native frame does not allow mapping to I444.
1718 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1719
1720 rtc::Event frame_destroyed_event;
1721 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1722 1, &frame_destroyed_event, codec_width_, codec_height_));
1723 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001724 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001725 fake_encoder_.GetLastInputPixelFormat());
1726 video_stream_encoder_->Stop();
1727}
1728
Henrik Boström56db9ff2021-03-24 09:06:45 +01001729TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001730 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001731 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001732
1733 rtc::Event frame_destroyed_event;
1734 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1735 1, &frame_destroyed_event, codec_width_, codec_height_));
1736 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001737 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001738 fake_encoder_.GetLastInputPixelFormat());
1739 video_stream_encoder_->Stop();
1740}
1741
Ying Wang9b881ab2020-02-07 14:29:32 +01001742TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001743 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001744 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001745 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1746 WaitForEncodedFrame(1);
1747
Henrik Boström381d1092020-05-12 18:49:07 +02001748 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001749 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001750 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1751 // frames. Adding two frames means that the first frame will be dropped and
1752 // the second frame will be sent to the encoder.
1753 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1754 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1755 WaitForEncodedFrame(3);
1756 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1757 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1758 WaitForEncodedFrame(5);
1759 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1760 video_stream_encoder_->Stop();
1761}
1762
mflodmancc3d4422017-08-03 08:27:51 -07001763TEST_F(VideoStreamEncoderTest,
1764 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001765 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001766 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001767 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001768
1769 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001770 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001771 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001772 // The encoder will have been configured once when the first frame is
1773 // received.
1774 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001775
1776 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001777 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001778 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001779 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001780 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001781
1782 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001783 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001784 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001785 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001786 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001787
mflodmancc3d4422017-08-03 08:27:51 -07001788 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001789}
1790
mflodmancc3d4422017-08-03 08:27:51 -07001791TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001792 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001793 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001794
1795 // Capture a frame and wait for it to synchronize with the encoder thread.
1796 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001797 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001798 // The encoder will have been configured once.
1799 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001800 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1801 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-02 23:45:26 -07001802
1803 codec_width_ *= 2;
1804 codec_height_ *= 2;
1805 // Capture a frame with a higher resolution and wait for it to synchronize
1806 // with the encoder thread.
1807 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001808 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 10:07:11 +02001809 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1810 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 21:37:57 +01001811 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001812
mflodmancc3d4422017-08-03 08:27:51 -07001813 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001814}
1815
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001816TEST_F(VideoStreamEncoderTest,
1817 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001818 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001819 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001820
1821 // Capture a frame and wait for it to synchronize with the encoder thread.
1822 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1823 WaitForEncodedFrame(1);
1824
1825 VideoEncoderConfig video_encoder_config;
1826 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1827 // Changing the max payload data length recreates encoder.
1828 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1829 kMaxPayloadLength / 2);
1830
1831 // Capture a frame and wait for it to synchronize with the encoder thread.
1832 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1833 WaitForEncodedFrame(2);
1834 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1835
1836 video_stream_encoder_->Stop();
1837}
1838
Sergey Silkin5ee69672019-07-02 14:18:34 +02001839TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001840 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001841 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001842
1843 VideoEncoderConfig video_encoder_config;
1844 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02001845 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1846 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001847 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1848 kMaxPayloadLength);
1849
1850 // Capture a frame and wait for it to synchronize with the encoder thread.
1851 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1852 WaitForEncodedFrame(1);
1853 // The encoder will have been configured once when the first frame is
1854 // received.
1855 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001856 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001857 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001858 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001859 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1860
Sergey Silkin6456e352019-07-08 17:56:40 +02001861 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1862 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 10:07:11 +02001863 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1864 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001865 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1866 kMaxPayloadLength);
1867
1868 // Capture a frame and wait for it to synchronize with the encoder thread.
1869 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1870 WaitForEncodedFrame(2);
1871 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1872 // Bitrate limits have changed - rate allocator should be reconfigured,
1873 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 10:07:11 +02001874 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001875 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001876 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001877 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001878 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001879
1880 video_stream_encoder_->Stop();
1881}
1882
Sergey Silkin6456e352019-07-08 17:56:40 +02001883TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001884 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001885 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001886 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001887
Sergey Silkincd02eba2020-01-20 14:48:40 +01001888 const uint32_t kMinEncBitrateKbps = 100;
1889 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001890 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001891 /*frame_size_pixels=*/codec_width_ * codec_height_,
1892 /*min_start_bitrate_bps=*/0,
1893 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1894 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001895 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1896
Sergey Silkincd02eba2020-01-20 14:48:40 +01001897 VideoEncoderConfig video_encoder_config;
1898 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1899 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1900 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1901 (kMinEncBitrateKbps + 1) * 1000;
1902 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1903 kMaxPayloadLength);
1904
1905 // When both encoder and app provide bitrate limits, the intersection of
1906 // provided sets should be used.
1907 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1908 WaitForEncodedFrame(1);
1909 EXPECT_EQ(kMaxEncBitrateKbps,
1910 bitrate_allocator_factory_.codec_config().maxBitrate);
1911 EXPECT_EQ(kMinEncBitrateKbps + 1,
1912 bitrate_allocator_factory_.codec_config().minBitrate);
1913
1914 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1915 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1916 (kMinEncBitrateKbps - 1) * 1000;
1917 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1918 kMaxPayloadLength);
1919 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001920 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001921 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001922 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001923 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001924 bitrate_allocator_factory_.codec_config().minBitrate);
1925
Sergey Silkincd02eba2020-01-20 14:48:40 +01001926 video_stream_encoder_->Stop();
1927}
1928
1929TEST_F(VideoStreamEncoderTest,
1930 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001931 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001932 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001933
1934 const uint32_t kMinAppBitrateKbps = 100;
1935 const uint32_t kMaxAppBitrateKbps = 200;
1936 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
1937 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
1938 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
1939 /*frame_size_pixels=*/codec_width_ * codec_height_,
1940 /*min_start_bitrate_bps=*/0,
1941 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1942 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
1943 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1944
1945 VideoEncoderConfig video_encoder_config;
1946 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1947 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
1948 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1949 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001950 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1951 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001952
Sergey Silkincd02eba2020-01-20 14:48:40 +01001953 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1954 WaitForEncodedFrame(1);
1955 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001956 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001957 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001958 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02001959
1960 video_stream_encoder_->Stop();
1961}
1962
1963TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001964 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02001965 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001966 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001967
1968 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001969 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001970 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001971 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001972 fake_encoder_.SetResolutionBitrateLimits(
1973 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
1974
1975 VideoEncoderConfig video_encoder_config;
1976 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1977 video_encoder_config.max_bitrate_bps = 0;
1978 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1979 kMaxPayloadLength);
1980
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001981 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001982 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
1983 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001984 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
1985 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001986 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
1987 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1988
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001989 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02001990 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
1991 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001992 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
1993 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02001994 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
1995 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
1996
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001997 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02001998 // encoder for 360p should be used.
1999 video_source_.IncomingCapturedFrame(
2000 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2001 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002002 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2003 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002004 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2005 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2006
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002007 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02002008 // ignored.
2009 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2010 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002011 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2012 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002013 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2014 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002015 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2016 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002017 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2018 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2019
2020 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2021 // for 270p should be used.
2022 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2023 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002024 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2025 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002026 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2027 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2028
2029 video_stream_encoder_->Stop();
2030}
2031
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002032TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002033 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002034 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002035
2036 VideoEncoderConfig video_encoder_config;
2037 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2038 video_encoder_config.max_bitrate_bps = 0;
2039 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2040 kMaxPayloadLength);
2041
2042 // Encode 720p frame to get the default encoder target bitrate.
2043 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2044 WaitForEncodedFrame(1);
2045 const uint32_t kDefaultTargetBitrateFor720pKbps =
2046 bitrate_allocator_factory_.codec_config()
2047 .simulcastStream[0]
2048 .targetBitrate;
2049
2050 // Set the max recommended encoder bitrate to something lower than the default
2051 // target bitrate.
2052 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2053 1280 * 720, 10 * 1000, 10 * 1000,
2054 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2055 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2056
2057 // Change resolution to trigger encoder reinitialization.
2058 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2059 WaitForEncodedFrame(2);
2060 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2061 WaitForEncodedFrame(3);
2062
2063 // Ensure the target bitrate is capped by the max bitrate.
2064 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2065 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2066 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2067 .simulcastStream[0]
2068 .targetBitrate *
2069 1000,
2070 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2071
2072 video_stream_encoder_->Stop();
2073}
2074
Åsa Perssona7e34d32021-01-20 15:36:13 +01002075TEST_F(VideoStreamEncoderTest,
2076 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2077 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2078 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2079 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2080 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2081 fake_encoder_.SetResolutionBitrateLimits(
2082 {kEncoderLimits270p, kEncoderLimits360p});
2083
2084 // Two streams, highest stream active.
2085 VideoEncoderConfig config;
2086 const int kNumStreams = 2;
2087 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2088 config.max_bitrate_bps = 0;
2089 config.simulcast_layers[0].active = false;
2090 config.simulcast_layers[1].active = true;
2091 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002092 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002093 "VP8", /*max qp*/ 56, /*screencast*/ false,
2094 /*screenshare enabled*/ false);
2095 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2096
2097 // The encoder bitrate limits for 270p should be used.
2098 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2099 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002100 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002101 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002102 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002103 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002104 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002105
2106 // The encoder bitrate limits for 360p should be used.
2107 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2108 EXPECT_FALSE(WaitForFrame(1000));
2109 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002110 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002111 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002112 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002113
2114 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2115 video_source_.IncomingCapturedFrame(
2116 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2117 EXPECT_FALSE(WaitForFrame(1000));
2118 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002119 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002120 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002121 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002122
2123 // Resolution higher than 360p. Encoder limits should be ignored.
2124 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2125 EXPECT_FALSE(WaitForFrame(1000));
2126 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002127 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002128 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002129 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002130 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002131 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002132 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002133 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002134
2135 // Resolution lower than 270p. The encoder limits for 270p should be used.
2136 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2137 EXPECT_FALSE(WaitForFrame(1000));
2138 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002139 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002140 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002141 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002142
2143 video_stream_encoder_->Stop();
2144}
2145
2146TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002147 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2148 // Two streams, highest stream active.
2149 VideoEncoderConfig config;
2150 const int kNumStreams = 2;
2151 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2152 config.max_bitrate_bps = 0;
2153 config.simulcast_layers[0].active = false;
2154 config.simulcast_layers[1].active = true;
2155 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002156 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002157 "VP8", /*max qp*/ 56, /*screencast*/ false,
2158 /*screenshare enabled*/ false);
2159 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2160
2161 // Default bitrate limits for 270p should be used.
2162 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2163 kDefaultLimits270p =
2164 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002165 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002166 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2167 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002168 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 10:39:51 +01002169 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002170 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002171 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002172 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002173
2174 // Default bitrate limits for 360p should be used.
2175 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2176 kDefaultLimits360p =
2177 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002178 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002179 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2180 EXPECT_FALSE(WaitForFrame(1000));
2181 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002182 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002183 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002184 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002185
2186 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2187 video_source_.IncomingCapturedFrame(
2188 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2189 EXPECT_FALSE(WaitForFrame(1000));
2190 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002191 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002192 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002193 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002194
2195 // Default bitrate limits for 540p should be used.
2196 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2197 kDefaultLimits540p =
2198 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002199 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002200 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2201 EXPECT_FALSE(WaitForFrame(1000));
2202 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002203 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002204 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002205 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002206
2207 video_stream_encoder_->Stop();
2208}
2209
2210TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002211 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2212 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2213 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2214 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2215 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2216 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2217 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2218 fake_encoder_.SetResolutionBitrateLimits(
2219 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2220
2221 // Three streams, middle stream active.
2222 VideoEncoderConfig config;
2223 const int kNumStreams = 3;
2224 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2225 config.simulcast_layers[0].active = false;
2226 config.simulcast_layers[1].active = true;
2227 config.simulcast_layers[2].active = false;
2228 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002229 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002230 "VP8", /*max qp*/ 56, /*screencast*/ false,
2231 /*screenshare enabled*/ false);
2232 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2233
2234 // The encoder bitrate limits for 360p should be used.
2235 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2236 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002237 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002238 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002239 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002240 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002241 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002242
2243 // The encoder bitrate limits for 270p should be used.
2244 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2245 EXPECT_FALSE(WaitForFrame(1000));
2246 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002247 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002248 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002249 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002250
2251 video_stream_encoder_->Stop();
2252}
2253
2254TEST_F(VideoStreamEncoderTest,
2255 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2256 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2257 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2258 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2259 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2260 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2261 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2262 fake_encoder_.SetResolutionBitrateLimits(
2263 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2264
2265 // Three streams, lowest stream active.
2266 VideoEncoderConfig config;
2267 const int kNumStreams = 3;
2268 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2269 config.simulcast_layers[0].active = true;
2270 config.simulcast_layers[1].active = false;
2271 config.simulcast_layers[2].active = false;
2272 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002273 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002274 "VP8", /*max qp*/ 56, /*screencast*/ false,
2275 /*screenshare enabled*/ false);
2276 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2277
2278 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2279 // on lowest stream, limits for 270p should not be used
2280 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2281 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002282 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002283 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002284 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002285 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002286 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002287
2288 video_stream_encoder_->Stop();
2289}
2290
2291TEST_F(VideoStreamEncoderTest,
2292 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2293 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2294 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2295 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2296 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2297 fake_encoder_.SetResolutionBitrateLimits(
2298 {kEncoderLimits270p, kEncoderLimits360p});
2299 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2300
2301 // Two streams, highest stream active.
2302 VideoEncoderConfig config;
2303 const int kNumStreams = 2;
2304 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2305 config.simulcast_layers[0].active = false;
2306 config.simulcast_layers[1].active = true;
2307 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2308 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002309 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002310 "VP8", /*max qp*/ 56, /*screencast*/ false,
2311 /*screenshare enabled*/ false);
2312 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2313
2314 // The encoder bitrate limits for 270p should be used.
2315 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2316 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002317 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002318 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002319 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002320 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002321 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002322
2323 // The max configured bitrate is less than the encoder limit for 360p.
2324 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2325 EXPECT_FALSE(WaitForFrame(1000));
2326 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002327 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002328 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002329 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002330
2331 video_stream_encoder_->Stop();
2332}
2333
mflodmancc3d4422017-08-03 08:27:51 -07002334TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002335 EXPECT_TRUE(video_source_.has_sinks());
2336 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002337 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002338 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002339 EXPECT_FALSE(video_source_.has_sinks());
2340 EXPECT_TRUE(new_video_source.has_sinks());
2341
mflodmancc3d4422017-08-03 08:27:51 -07002342 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002343}
2344
mflodmancc3d4422017-08-03 08:27:51 -07002345TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002346 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002347 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002348 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002349 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002350}
2351
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002352class ResolutionAlignmentTest
2353 : public VideoStreamEncoderTest,
2354 public ::testing::WithParamInterface<
2355 ::testing::tuple<int, std::vector<double>>> {
2356 public:
2357 ResolutionAlignmentTest()
2358 : requested_alignment_(::testing::get<0>(GetParam())),
2359 scale_factors_(::testing::get<1>(GetParam())) {}
2360
2361 protected:
2362 const int requested_alignment_;
2363 const std::vector<double> scale_factors_;
2364};
2365
2366INSTANTIATE_TEST_SUITE_P(
2367 AlignmentAndScaleFactors,
2368 ResolutionAlignmentTest,
2369 ::testing::Combine(
2370 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2371 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2372 std::vector<double>{-1.0, -1.0},
2373 std::vector<double>{-1.0, -1.0, -1.0},
2374 std::vector<double>{4.0, 2.0, 1.0},
2375 std::vector<double>{9999.0, -1.0, 1.0},
2376 std::vector<double>{3.99, 2.01, 1.0},
2377 std::vector<double>{4.9, 1.7, 1.25},
2378 std::vector<double>{10.0, 4.0, 3.0},
2379 std::vector<double>{1.75, 3.5},
2380 std::vector<double>{1.5, 2.5},
2381 std::vector<double>{1.3, 1.0})));
2382
2383TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2384 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002385 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002386 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2387 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2388
2389 // Fill config with the scaling factor by which to reduce encoding size.
2390 const int num_streams = scale_factors_.size();
2391 VideoEncoderConfig config;
2392 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2393 for (int i = 0; i < num_streams; ++i) {
2394 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2395 }
2396 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002397 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002398 "VP8", /*max qp*/ 56, /*screencast*/ false,
2399 /*screenshare enabled*/ false);
2400 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2401
Henrik Boström381d1092020-05-12 18:49:07 +02002402 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002403 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2404 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002405 // Wait for all layers before triggering event.
2406 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002407
2408 // On the 1st frame, we should have initialized the encoder and
2409 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002410 int64_t timestamp_ms = kFrameIntervalMs;
2411 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2412 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002413 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002414
2415 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2416 // (It's up the to the encoder to potentially drop the previous frame,
2417 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002418 timestamp_ms += kFrameIntervalMs;
2419 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2420 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002421 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002422
Asa Persson606d3cb2021-10-04 10:07:11 +02002423 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002424 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2425 // Frame size should be a multiple of the requested alignment.
2426 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2427 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2428 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2429 // Aspect ratio should match.
2430 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2431 codec.height * codec.simulcastStream[i].width);
2432 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002433
2434 video_stream_encoder_->Stop();
2435}
2436
Jonathan Yubc771b72017-12-08 17:04:29 -08002437TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2438 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002439 const int kWidth = 1280;
2440 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002441
2442 // We rely on the automatic resolution adaptation, but we handle framerate
2443 // adaptation manually by mocking the stats proxy.
2444 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002445
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002446 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002447 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002448 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002449 video_stream_encoder_->SetSource(&video_source_,
2450 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002451 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002452 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002453 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002454 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2455
Jonathan Yubc771b72017-12-08 17:04:29 -08002456 // Adapt down as far as possible.
2457 rtc::VideoSinkWants last_wants;
2458 int64_t t = 1;
2459 int loop_count = 0;
2460 do {
2461 ++loop_count;
2462 last_wants = video_source_.sink_wants();
2463
2464 // Simulate the framerate we've been asked to adapt to.
2465 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2466 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2467 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2468 mock_stats.input_frame_rate = fps;
2469 stats_proxy_->SetMockStats(mock_stats);
2470
2471 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2472 sink_.WaitForEncodedFrame(t);
2473 t += frame_interval_ms;
2474
mflodmancc3d4422017-08-03 08:27:51 -07002475 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002476 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002477 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002478 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2479 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002480 } while (video_source_.sink_wants().max_pixel_count <
2481 last_wants.max_pixel_count ||
2482 video_source_.sink_wants().max_framerate_fps <
2483 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002484
Jonathan Yubc771b72017-12-08 17:04:29 -08002485 // Verify that we've adapted all the way down.
2486 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002487 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002488 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2489 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002490 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002491 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2492 *video_source_.last_sent_height());
2493 EXPECT_EQ(kMinBalancedFramerateFps,
2494 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002495
Jonathan Yubc771b72017-12-08 17:04:29 -08002496 // Adapt back up the same number of times we adapted down.
2497 for (int i = 0; i < loop_count - 1; ++i) {
2498 last_wants = video_source_.sink_wants();
2499
2500 // Simulate the framerate we've been asked to adapt to.
2501 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2502 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2503 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2504 mock_stats.input_frame_rate = fps;
2505 stats_proxy_->SetMockStats(mock_stats);
2506
2507 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2508 sink_.WaitForEncodedFrame(t);
2509 t += frame_interval_ms;
2510
Henrik Boström91aa7322020-04-28 12:24:33 +02002511 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002512 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002513 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002514 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2515 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002516 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2517 last_wants.max_pixel_count ||
2518 video_source_.sink_wants().max_framerate_fps >
2519 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002520 }
2521
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002522 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002523 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002524 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002525 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2526 EXPECT_EQ((loop_count - 1) * 2,
2527 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002528
mflodmancc3d4422017-08-03 08:27:51 -07002529 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002530}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002531
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002532TEST_F(VideoStreamEncoderTest,
2533 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02002534 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2535 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002536 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002537
2538 const int kFrameWidth = 1280;
2539 const int kFrameHeight = 720;
2540
2541 int64_t ntp_time = kFrameIntervalMs;
2542
2543 // Force an input frame rate to be available, or the adaptation call won't
2544 // know what framerate to adapt form.
2545 const int kInputFps = 30;
2546 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2547 stats.input_frame_rate = kInputFps;
2548 stats_proxy_->SetMockStats(stats);
2549
2550 video_source_.set_adaptation_enabled(true);
2551 video_stream_encoder_->SetSource(
2552 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002553 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002554 video_source_.IncomingCapturedFrame(
2555 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2556 sink_.WaitForEncodedFrame(ntp_time);
2557 ntp_time += kFrameIntervalMs;
2558
2559 // Trigger CPU overuse.
2560 video_stream_encoder_->TriggerCpuOveruse();
2561 video_source_.IncomingCapturedFrame(
2562 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2563 sink_.WaitForEncodedFrame(ntp_time);
2564 ntp_time += kFrameIntervalMs;
2565
2566 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2567 EXPECT_EQ(std::numeric_limits<int>::max(),
2568 video_source_.sink_wants().max_pixel_count);
2569 // Some framerate constraint should be set.
2570 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2571 EXPECT_LT(restricted_fps, kInputFps);
2572 video_source_.IncomingCapturedFrame(
2573 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2574 sink_.WaitForEncodedFrame(ntp_time);
2575 ntp_time += 100;
2576
Henrik Boström2671dac2020-05-19 16:29:09 +02002577 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002578 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2579 // Give the encoder queue time to process the change in degradation preference
2580 // by waiting for an encoded frame.
2581 video_source_.IncomingCapturedFrame(
2582 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2583 sink_.WaitForEncodedFrame(ntp_time);
2584 ntp_time += kFrameIntervalMs;
2585
2586 video_stream_encoder_->TriggerQualityLow();
2587 video_source_.IncomingCapturedFrame(
2588 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2589 sink_.WaitForEncodedFrame(ntp_time);
2590 ntp_time += kFrameIntervalMs;
2591
2592 // Some resolution constraint should be set.
2593 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2594 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2595 kFrameWidth * kFrameHeight);
2596 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2597
2598 int pixel_count = video_source_.sink_wants().max_pixel_count;
2599 // Triggering a CPU underuse should not change the sink wants since it has
2600 // not been overused for resolution since we changed degradation preference.
2601 video_stream_encoder_->TriggerCpuUnderuse();
2602 video_source_.IncomingCapturedFrame(
2603 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2604 sink_.WaitForEncodedFrame(ntp_time);
2605 ntp_time += kFrameIntervalMs;
2606 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2607 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2608
Evan Shrubsole64469032020-06-11 10:45:29 +02002609 // Change the degradation preference back. CPU underuse should not adapt since
2610 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002611 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002612 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2613 video_source_.IncomingCapturedFrame(
2614 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2615 sink_.WaitForEncodedFrame(ntp_time);
2616 ntp_time += 100;
2617 // Resolution adaptations is gone after changing degradation preference.
2618 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2619 EXPECT_EQ(std::numeric_limits<int>::max(),
2620 video_source_.sink_wants().max_pixel_count);
2621 // The fps adaptation from above is now back.
2622 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2623
2624 // Trigger CPU underuse.
2625 video_stream_encoder_->TriggerCpuUnderuse();
2626 video_source_.IncomingCapturedFrame(
2627 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2628 sink_.WaitForEncodedFrame(ntp_time);
2629 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002630 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2631
2632 // Trigger QP underuse, fps should return to normal.
2633 video_stream_encoder_->TriggerQualityHigh();
2634 video_source_.IncomingCapturedFrame(
2635 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2636 sink_.WaitForEncodedFrame(ntp_time);
2637 ntp_time += kFrameIntervalMs;
2638 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002639
2640 video_stream_encoder_->Stop();
2641}
2642
mflodmancc3d4422017-08-03 08:27:51 -07002643TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002644 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002645 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002646 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002647
sprangc5d62e22017-04-02 23:53:04 -07002648 const int kFrameWidth = 1280;
2649 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002650
Åsa Persson8c1bf952018-09-13 10:42:19 +02002651 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002652
kthelgason5e13d412016-12-01 03:59:51 -08002653 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002654 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002655 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002656 frame_timestamp += kFrameIntervalMs;
2657
perkj803d97f2016-11-01 11:45:46 -07002658 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002659 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002660 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002661 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002662 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002663 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002664
asapersson0944a802017-04-07 00:57:58 -07002665 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002666 // wanted resolution.
2667 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2668 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2669 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002670 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002671
2672 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002673 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002674 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002675 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002676 // Give the encoder queue time to process the change in degradation preference
2677 // by waiting for an encoded frame.
2678 new_video_source.IncomingCapturedFrame(
2679 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2680 sink_.WaitForEncodedFrame(frame_timestamp);
2681 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002682 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002683 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002684
sprangc5d62e22017-04-02 23:53:04 -07002685 // Force an input frame rate to be available, or the adaptation call won't
2686 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002687 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002688 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002689 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002690 stats_proxy_->SetMockStats(stats);
2691
mflodmancc3d4422017-08-03 08:27:51 -07002692 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002693 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002694 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002695 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002696 frame_timestamp += kFrameIntervalMs;
2697
2698 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002699 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002700 EXPECT_EQ(std::numeric_limits<int>::max(),
2701 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002702 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002703
asapersson02465b82017-04-10 01:12:52 -07002704 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002705 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2706 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002707 // Give the encoder queue time to process the change in degradation preference
2708 // by waiting for an encoded frame.
2709 new_video_source.IncomingCapturedFrame(
2710 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2711 sink_.WaitForEncodedFrame(frame_timestamp);
2712 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002713 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002714
mflodmancc3d4422017-08-03 08:27:51 -07002715 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002716 new_video_source.IncomingCapturedFrame(
2717 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002718 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002719 frame_timestamp += kFrameIntervalMs;
2720
2721 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002722 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002723
2724 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002725 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002726 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002727 // Give the encoder queue time to process the change in degradation preference
2728 // by waiting for an encoded frame.
2729 new_video_source.IncomingCapturedFrame(
2730 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2731 sink_.WaitForEncodedFrame(frame_timestamp);
2732 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002733 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2734 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002735 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002736 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002737
2738 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002739 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002740 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002741 // Give the encoder queue time to process the change in degradation preference
2742 // by waiting for an encoded frame.
2743 new_video_source.IncomingCapturedFrame(
2744 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2745 sink_.WaitForEncodedFrame(frame_timestamp);
2746 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002747 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2748 EXPECT_EQ(std::numeric_limits<int>::max(),
2749 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002750 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002751
mflodmancc3d4422017-08-03 08:27:51 -07002752 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002753}
2754
mflodmancc3d4422017-08-03 08:27:51 -07002755TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002756 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002757 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002758
asaperssonfab67072017-04-04 05:51:49 -07002759 const int kWidth = 1280;
2760 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002761 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002762 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002763 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2764 EXPECT_FALSE(stats.bw_limited_resolution);
2765 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2766
2767 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002768 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002769 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002770 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002771
2772 stats = stats_proxy_->GetStats();
2773 EXPECT_TRUE(stats.bw_limited_resolution);
2774 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2775
2776 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002777 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002778 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002779 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002780
2781 stats = stats_proxy_->GetStats();
2782 EXPECT_FALSE(stats.bw_limited_resolution);
2783 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2784 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2785
mflodmancc3d4422017-08-03 08:27:51 -07002786 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002787}
2788
mflodmancc3d4422017-08-03 08:27:51 -07002789TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002790 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002791 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002792
2793 const int kWidth = 1280;
2794 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002795 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002796 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002797 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2798 EXPECT_FALSE(stats.cpu_limited_resolution);
2799 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2800
2801 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002802 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002803 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002804 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002805
2806 stats = stats_proxy_->GetStats();
2807 EXPECT_TRUE(stats.cpu_limited_resolution);
2808 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2809
2810 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002811 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002812 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002813 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002814
2815 stats = stats_proxy_->GetStats();
2816 EXPECT_FALSE(stats.cpu_limited_resolution);
2817 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002818 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002819
mflodmancc3d4422017-08-03 08:27:51 -07002820 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002821}
2822
mflodmancc3d4422017-08-03 08:27:51 -07002823TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002824 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002825 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002826
asaperssonfab67072017-04-04 05:51:49 -07002827 const int kWidth = 1280;
2828 const int kHeight = 720;
2829 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002830 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002831 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002832 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002833 EXPECT_FALSE(stats.cpu_limited_resolution);
2834 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2835
asaperssonfab67072017-04-04 05:51:49 -07002836 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002837 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002838 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002839 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002840 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002841 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002842 EXPECT_TRUE(stats.cpu_limited_resolution);
2843 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2844
2845 // Set new source with adaptation still enabled.
2846 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002847 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002848 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002849
asaperssonfab67072017-04-04 05:51:49 -07002850 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002851 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002852 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002853 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002854 EXPECT_TRUE(stats.cpu_limited_resolution);
2855 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2856
2857 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002858 video_stream_encoder_->SetSource(&new_video_source,
2859 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002860
asaperssonfab67072017-04-04 05:51:49 -07002861 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002862 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002863 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002864 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002865 EXPECT_FALSE(stats.cpu_limited_resolution);
2866 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2867
2868 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002869 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002870 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002871
asaperssonfab67072017-04-04 05:51:49 -07002872 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002873 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002874 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002875 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002876 EXPECT_TRUE(stats.cpu_limited_resolution);
2877 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2878
asaperssonfab67072017-04-04 05:51:49 -07002879 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002880 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002881 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002882 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002883 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002884 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002885 EXPECT_FALSE(stats.cpu_limited_resolution);
2886 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002887 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002888
mflodmancc3d4422017-08-03 08:27:51 -07002889 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002890}
2891
mflodmancc3d4422017-08-03 08:27:51 -07002892TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002893 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002894 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002895
asaperssonfab67072017-04-04 05:51:49 -07002896 const int kWidth = 1280;
2897 const int kHeight = 720;
2898 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002899 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002900 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002901 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002902 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002903 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002904
2905 // Set new source with adaptation still enabled.
2906 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002907 video_stream_encoder_->SetSource(&new_video_source,
2908 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002909
asaperssonfab67072017-04-04 05:51:49 -07002910 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002911 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002912 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002913 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002914 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002915 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002916
asaperssonfab67072017-04-04 05:51:49 -07002917 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002918 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002919 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002920 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002921 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002922 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002923 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002924 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002925
asaperssonfab67072017-04-04 05:51:49 -07002926 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002927 video_stream_encoder_->SetSource(&new_video_source,
2928 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002929
asaperssonfab67072017-04-04 05:51:49 -07002930 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002931 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002932 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002933 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002934 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002935 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002936
asapersson02465b82017-04-10 01:12:52 -07002937 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07002938 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002939 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08002940
asaperssonfab67072017-04-04 05:51:49 -07002941 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002942 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002943 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002944 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002945 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002946 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2947 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002948
mflodmancc3d4422017-08-03 08:27:51 -07002949 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002950}
2951
mflodmancc3d4422017-08-03 08:27:51 -07002952TEST_F(VideoStreamEncoderTest,
2953 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02002954 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002955 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07002956
2957 const int kWidth = 1280;
2958 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02002959 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07002960 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002961 video_source_.IncomingCapturedFrame(
2962 CreateFrame(timestamp_ms, kWidth, kHeight));
2963 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002964 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2965 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
2966 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2967
2968 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002969 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002970 timestamp_ms += kFrameIntervalMs;
2971 video_source_.IncomingCapturedFrame(
2972 CreateFrame(timestamp_ms, kWidth, kHeight));
2973 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002974 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
2975 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2976 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2977
2978 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002979 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02002980 timestamp_ms += kFrameIntervalMs;
2981 video_source_.IncomingCapturedFrame(
2982 CreateFrame(timestamp_ms, kWidth, kHeight));
2983 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07002984 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
2985 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
2986 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2987
Niels Möller4db138e2018-04-19 09:04:13 +02002988 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07002989 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02002990
2991 VideoEncoderConfig video_encoder_config;
2992 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2993 // Make format different, to force recreation of encoder.
2994 video_encoder_config.video_format.parameters["foo"] = "foo";
2995 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02002996 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002997 timestamp_ms += kFrameIntervalMs;
2998 video_source_.IncomingCapturedFrame(
2999 CreateFrame(timestamp_ms, kWidth, kHeight));
3000 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003001 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3002 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3003 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3004
mflodmancc3d4422017-08-03 08:27:51 -07003005 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003006}
3007
mflodmancc3d4422017-08-03 08:27:51 -07003008TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003009 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003010 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003011 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003012
3013 const int kWidth = 1280;
3014 const int kHeight = 720;
3015 int sequence = 1;
3016
3017 // Enable BALANCED preference, no initial limitation.
3018 test::FrameForwarder source;
3019 video_stream_encoder_->SetSource(&source,
3020 webrtc::DegradationPreference::BALANCED);
3021 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3022 WaitForEncodedFrame(sequence++);
3023 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3024 EXPECT_FALSE(stats.cpu_limited_resolution);
3025 EXPECT_FALSE(stats.cpu_limited_framerate);
3026 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3027
3028 // Trigger CPU overuse, should now adapt down.
3029 video_stream_encoder_->TriggerCpuOveruse();
3030 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3031 WaitForEncodedFrame(sequence++);
3032 stats = stats_proxy_->GetStats();
3033 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3034
3035 // Set new degradation preference should clear restrictions since we changed
3036 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003037 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003038 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3039 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3040 WaitForEncodedFrame(sequence++);
3041 stats = stats_proxy_->GetStats();
3042 EXPECT_FALSE(stats.cpu_limited_resolution);
3043 EXPECT_FALSE(stats.cpu_limited_framerate);
3044 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3045
3046 // Force an input frame rate to be available, or the adaptation call won't
3047 // know what framerate to adapt from.
3048 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3049 mock_stats.input_frame_rate = 30;
3050 stats_proxy_->SetMockStats(mock_stats);
3051 video_stream_encoder_->TriggerCpuOveruse();
3052 stats_proxy_->ResetMockStats();
3053 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3054 WaitForEncodedFrame(sequence++);
3055
3056 // We have now adapted once.
3057 stats = stats_proxy_->GetStats();
3058 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3059
3060 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003061 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3062 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003063 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3064 WaitForEncodedFrame(sequence++);
3065 stats = stats_proxy_->GetStats();
3066 EXPECT_FALSE(stats.cpu_limited_resolution);
3067 EXPECT_FALSE(stats.cpu_limited_framerate);
3068 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3069
3070 video_stream_encoder_->Stop();
3071}
3072
3073TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003074 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003075 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003076 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003077
asapersson0944a802017-04-07 00:57:58 -07003078 const int kWidth = 1280;
3079 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003080 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003081
asaperssonfab67072017-04-04 05:51:49 -07003082 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003083 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003084 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003085 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003086 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003087 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3088
asapersson02465b82017-04-10 01:12:52 -07003089 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003090 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003091 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003092 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003093 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003094 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003095 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003096 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3097
3098 // Set new source with adaptation still enabled.
3099 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003100 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003101 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003102
3103 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003104 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003105 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003106 stats = stats_proxy_->GetStats();
3107 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003108 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003109 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3110
sprangc5d62e22017-04-02 23:53:04 -07003111 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003112 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003113 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003114 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003115 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003116 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003117 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003118 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003119 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003120 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003121 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3122
sprangc5d62e22017-04-02 23:53:04 -07003123 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003124 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003125 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3126 mock_stats.input_frame_rate = 30;
3127 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003128 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003129 stats_proxy_->ResetMockStats();
3130
3131 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003132 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003133 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003134
3135 // Framerate now adapted.
3136 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003137 EXPECT_FALSE(stats.cpu_limited_resolution);
3138 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003139 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3140
3141 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003142 video_stream_encoder_->SetSource(&new_video_source,
3143 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003144 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003145 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003146 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003147
3148 stats = stats_proxy_->GetStats();
3149 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003150 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003151 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3152
3153 // Try to trigger overuse. Should not succeed.
3154 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003155 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003156 stats_proxy_->ResetMockStats();
3157
3158 stats = stats_proxy_->GetStats();
3159 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003160 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003161 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3162
3163 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003164 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003165 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003166 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003167 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003168 stats = stats_proxy_->GetStats();
3169 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003170 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003171 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003172
3173 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003174 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003175 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003176 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003177 stats = stats_proxy_->GetStats();
3178 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003179 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003180 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3181
3182 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003183 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003184 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003185 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003186 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003187 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003188 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003189 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003190 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003191 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003192 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3193
3194 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003195 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003196 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003197 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003198 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003199 stats = stats_proxy_->GetStats();
3200 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003201 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003202 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003203 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003204
mflodmancc3d4422017-08-03 08:27:51 -07003205 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003206}
3207
mflodmancc3d4422017-08-03 08:27:51 -07003208TEST_F(VideoStreamEncoderTest,
3209 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003210 const int kWidth = 1280;
3211 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003212 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003213 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003214
asaperssonfab67072017-04-04 05:51:49 -07003215 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003216 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003217
asaperssonfab67072017-04-04 05:51:49 -07003218 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003219 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003220
asaperssonfab67072017-04-04 05:51:49 -07003221 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003222 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003223
asaperssonfab67072017-04-04 05:51:49 -07003224 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003225 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003226
kthelgason876222f2016-11-29 01:44:11 -08003227 // Expect a scale down.
3228 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003229 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003230
asapersson02465b82017-04-10 01:12:52 -07003231 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003232 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003233 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003234 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003235
asaperssonfab67072017-04-04 05:51:49 -07003236 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003237 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003238 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003239 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003240
asaperssonfab67072017-04-04 05:51:49 -07003241 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003242 EXPECT_EQ(std::numeric_limits<int>::max(),
3243 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003244
asaperssonfab67072017-04-04 05:51:49 -07003245 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003246 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003247 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003248 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003249
asapersson02465b82017-04-10 01:12:52 -07003250 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003251 EXPECT_EQ(std::numeric_limits<int>::max(),
3252 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003253
mflodmancc3d4422017-08-03 08:27:51 -07003254 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003255}
3256
mflodmancc3d4422017-08-03 08:27:51 -07003257TEST_F(VideoStreamEncoderTest,
3258 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003259 const int kWidth = 1280;
3260 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003261 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003262 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003263
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003264 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003265 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003266 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003267 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003268
3269 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003270 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003271 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003272 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3273 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3274
3275 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003276 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003277 EXPECT_THAT(source.sink_wants(),
3278 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003279 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3280 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3281 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3282
3283 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003284 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003285 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3286 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3287 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3288
mflodmancc3d4422017-08-03 08:27:51 -07003289 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003290}
3291
mflodmancc3d4422017-08-03 08:27:51 -07003292TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003293 const int kWidth = 1280;
3294 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003295 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003296 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003297
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003298 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003299 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003300 video_stream_encoder_->SetSource(&source,
3301 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003302 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3303 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003304 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003305
3306 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003307 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003308 EXPECT_THAT(source.sink_wants(),
3309 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003310 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3311 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3312 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3313
3314 // Trigger adapt down for same input resolution, expect no change.
3315 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3316 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003317 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003318 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3319 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3320 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3321
3322 // Trigger adapt down for larger input resolution, expect no change.
3323 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3324 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003325 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003326 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3327 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3328 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3329
mflodmancc3d4422017-08-03 08:27:51 -07003330 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003331}
3332
mflodmancc3d4422017-08-03 08:27:51 -07003333TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003334 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3335 const int kWidth = 640;
3336 const int kHeight = 360;
3337 const int64_t kFrameIntervalMs = 150;
3338 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003339 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003340
3341 // Enable BALANCED preference, no initial limitation.
3342 AdaptingFrameForwarder source(&time_controller_);
3343 source.set_adaptation_enabled(true);
3344 video_stream_encoder_->SetSource(&source,
3345 webrtc::DegradationPreference::BALANCED);
3346
3347 int64_t timestamp_ms = kFrameIntervalMs;
3348 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3349 sink_.WaitForEncodedFrame(kWidth, kHeight);
3350 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3351 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3352 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3353 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3354
3355 // Trigger adapt down, expect reduced fps (640x360@15fps).
3356 video_stream_encoder_->TriggerQualityLow();
3357 timestamp_ms += kFrameIntervalMs;
3358 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3359 sink_.WaitForEncodedFrame(timestamp_ms);
3360 EXPECT_THAT(source.sink_wants(),
3361 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3362 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3363 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3364 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3365
3366 // Source requests 270p, expect reduced resolution (480x270@15fps).
3367 source.OnOutputFormatRequest(480, 270);
3368 timestamp_ms += kFrameIntervalMs;
3369 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3370 WaitForEncodedFrame(480, 270);
3371 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3372
3373 // Trigger adapt down, expect reduced fps (480x270@10fps).
3374 video_stream_encoder_->TriggerQualityLow();
3375 timestamp_ms += kFrameIntervalMs;
3376 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3377 sink_.WaitForEncodedFrame(timestamp_ms);
3378 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3379 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3380 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3381 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3382
3383 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3384 source.OnOutputFormatRequest(320, 180);
3385 timestamp_ms += kFrameIntervalMs;
3386 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3387 WaitForEncodedFrame(320, 180);
3388 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3389
3390 // Trigger adapt down, expect reduced fps (320x180@7fps).
3391 video_stream_encoder_->TriggerQualityLow();
3392 timestamp_ms += kFrameIntervalMs;
3393 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3394 sink_.WaitForEncodedFrame(timestamp_ms);
3395 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3396 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3397 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3398 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3399
3400 // Source requests VGA, expect increased resolution (640x360@7fps).
3401 source.OnOutputFormatRequest(640, 360);
3402 timestamp_ms += kFrameIntervalMs;
3403 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3404 WaitForEncodedFrame(timestamp_ms);
3405 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3406
3407 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3408 video_stream_encoder_->TriggerQualityHigh();
3409 timestamp_ms += kFrameIntervalMs;
3410 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3411 WaitForEncodedFrame(timestamp_ms);
3412 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3413 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3414 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3415 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3416
3417 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3418 video_stream_encoder_->TriggerQualityHigh();
3419 timestamp_ms += kFrameIntervalMs;
3420 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3421 WaitForEncodedFrame(timestamp_ms);
3422 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3423 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3424 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3425 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3426
3427 // Trigger adapt up, expect increased fps (640x360@maxfps).
3428 video_stream_encoder_->TriggerQualityHigh();
3429 timestamp_ms += kFrameIntervalMs;
3430 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3431 WaitForEncodedFrame(timestamp_ms);
3432 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3433 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3434 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3435 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3436
3437 video_stream_encoder_->Stop();
3438}
3439
3440TEST_F(VideoStreamEncoderTest,
3441 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3442 const int kWidth = 1280;
3443 const int kHeight = 720;
3444 const int64_t kFrameIntervalMs = 150;
3445 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003446 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003447
3448 // Enable BALANCED preference, no initial limitation.
3449 AdaptingFrameForwarder source(&time_controller_);
3450 source.set_adaptation_enabled(true);
3451 video_stream_encoder_->SetSource(&source,
3452 webrtc::DegradationPreference::BALANCED);
3453
3454 int64_t timestamp_ms = kFrameIntervalMs;
3455 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3456 sink_.WaitForEncodedFrame(kWidth, kHeight);
3457 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3458 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3459 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3460 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3461
3462 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3463 video_stream_encoder_->TriggerQualityLow();
3464 timestamp_ms += kFrameIntervalMs;
3465 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3466 sink_.WaitForEncodedFrame(timestamp_ms);
3467 EXPECT_THAT(source.sink_wants(),
3468 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3469 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3470 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3471 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3472
3473 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3474 video_stream_encoder_->TriggerQualityLow();
3475 timestamp_ms += kFrameIntervalMs;
3476 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3477 sink_.WaitForEncodedFrame(timestamp_ms);
3478 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3479 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3480 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3481 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3482
3483 // Trigger adapt down, expect reduced fps (640x360@15fps).
3484 video_stream_encoder_->TriggerQualityLow();
3485 timestamp_ms += kFrameIntervalMs;
3486 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3487 WaitForEncodedFrame(timestamp_ms);
3488 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3489 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3490 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3491 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3492
3493 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3494 source.OnOutputFormatRequest(320, 180);
3495 timestamp_ms += kFrameIntervalMs;
3496 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3497 WaitForEncodedFrame(320, 180);
3498 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3499 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3500
3501 // Trigger adapt down, expect reduced fps (320x180@7fps).
3502 video_stream_encoder_->TriggerCpuOveruse();
3503 timestamp_ms += kFrameIntervalMs;
3504 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3505 WaitForEncodedFrame(timestamp_ms);
3506 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3507 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3508 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3509 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3510 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3511 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3512 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3513
3514 // Source requests HD, expect increased resolution (640x360@7fps).
3515 source.OnOutputFormatRequest(1280, 720);
3516 timestamp_ms += kFrameIntervalMs;
3517 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3518 WaitForEncodedFrame(timestamp_ms);
3519 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3520 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3521
3522 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3523 video_stream_encoder_->TriggerCpuUnderuse();
3524 timestamp_ms += kFrameIntervalMs;
3525 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3526 WaitForEncodedFrame(timestamp_ms);
3527 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3528 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3529 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3530 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3531 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3532 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3533 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3534
3535 // Trigger adapt up, expect increased fps (640x360@maxfps).
3536 video_stream_encoder_->TriggerQualityHigh();
3537 video_stream_encoder_->TriggerCpuUnderuse();
3538 timestamp_ms += kFrameIntervalMs;
3539 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3540 WaitForEncodedFrame(timestamp_ms);
3541 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3542 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3543 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3544 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3545 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3546 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3547 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3548
3549 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3550 video_stream_encoder_->TriggerQualityHigh();
3551 video_stream_encoder_->TriggerCpuUnderuse();
3552 timestamp_ms += kFrameIntervalMs;
3553 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3554 WaitForEncodedFrame(timestamp_ms);
3555 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3556 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3557 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3558 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3559 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3560 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3561 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3562
3563 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3564 video_stream_encoder_->TriggerQualityHigh();
3565 video_stream_encoder_->TriggerCpuUnderuse();
3566 timestamp_ms += kFrameIntervalMs;
3567 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3568 WaitForEncodedFrame(timestamp_ms);
3569 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3570 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3571 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3572 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3573 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3574 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3575 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3576
3577 video_stream_encoder_->Stop();
3578}
3579
3580TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003581 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003582 const int kWidth = 1280;
3583 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003584 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003585 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003586
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003587 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003588 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003589 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003590 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003591
3592 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003593 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003594 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003595 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3596 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3597
3598 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003599 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003600 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003601 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3602 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3603
mflodmancc3d4422017-08-03 08:27:51 -07003604 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003605}
3606
mflodmancc3d4422017-08-03 08:27:51 -07003607TEST_F(VideoStreamEncoderTest,
3608 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003609 const int kWidth = 1280;
3610 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003611 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003612 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003613
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003614 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003615 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003616 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003617 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003618
3619 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003620 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003621 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003622 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003623 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3624
3625 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003626 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003627 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003628 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003629 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3630
mflodmancc3d4422017-08-03 08:27:51 -07003631 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003632}
3633
mflodmancc3d4422017-08-03 08:27:51 -07003634TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003635 const int kWidth = 1280;
3636 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003637 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003638 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003639
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003640 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003641 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003642 video_stream_encoder_->SetSource(&source,
3643 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003644
3645 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3646 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003647 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003648 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3649 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3650 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3651
3652 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003653 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003654 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003655 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3656 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3657 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3658
mflodmancc3d4422017-08-03 08:27:51 -07003659 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003660}
3661
mflodmancc3d4422017-08-03 08:27:51 -07003662TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003663 const int kWidth = 1280;
3664 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003665 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003666 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003667
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003668 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003669 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003670 video_stream_encoder_->SetSource(&source,
3671 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003672
3673 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3674 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003675 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003676 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3677 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3678 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3679
3680 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003681 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003682 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003683 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3684 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3685 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3686
mflodmancc3d4422017-08-03 08:27:51 -07003687 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003688}
3689
mflodmancc3d4422017-08-03 08:27:51 -07003690TEST_F(VideoStreamEncoderTest,
3691 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003692 const int kWidth = 1280;
3693 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003694 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003695 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003696
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003697 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003698 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003699 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003700 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003701 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003702
3703 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003704 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003705 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003706 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3707 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3708
3709 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003710 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003711 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003712 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003713 EXPECT_THAT(source.sink_wants(),
3714 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003715 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3716 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3717
3718 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003719 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003720 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003721 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3722 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3723 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3724
mflodmancc3d4422017-08-03 08:27:51 -07003725 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003726}
3727
mflodmancc3d4422017-08-03 08:27:51 -07003728TEST_F(VideoStreamEncoderTest,
3729 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003730 const int kWidth = 1280;
3731 const int kHeight = 720;
3732 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003733 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003734 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003735
3736 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3737 stats.input_frame_rate = kInputFps;
3738 stats_proxy_->SetMockStats(stats);
3739
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003740 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003741 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3742 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003743 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003744
3745 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003746 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003747 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3748 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003749 EXPECT_THAT(video_source_.sink_wants(),
3750 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003751
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003752 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003753 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003754 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003755 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003756 // Give the encoder queue time to process the change in degradation preference
3757 // by waiting for an encoded frame.
3758 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3759 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003760 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003761
3762 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003763 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003764 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3765 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003766 EXPECT_THAT(new_video_source.sink_wants(),
3767 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003768
3769 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003770 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003771 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003772
mflodmancc3d4422017-08-03 08:27:51 -07003773 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003774}
3775
mflodmancc3d4422017-08-03 08:27:51 -07003776TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003777 const int kWidth = 1280;
3778 const int kHeight = 720;
3779 const size_t kNumFrames = 10;
3780
Henrik Boström381d1092020-05-12 18:49:07 +02003781 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003782 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003783
asaperssond0de2952017-04-21 01:47:31 -07003784 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003785 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003786 video_source_.set_adaptation_enabled(true);
3787
3788 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3789 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3790
3791 int downscales = 0;
3792 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003793 video_source_.IncomingCapturedFrame(
3794 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3795 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003796
asaperssonfab67072017-04-04 05:51:49 -07003797 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003798 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003799 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003800 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003801
3802 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3803 ++downscales;
3804
3805 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3806 EXPECT_EQ(downscales,
3807 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3808 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003809 }
mflodmancc3d4422017-08-03 08:27:51 -07003810 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003811}
3812
mflodmancc3d4422017-08-03 08:27:51 -07003813TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003814 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3815 const int kWidth = 1280;
3816 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003817 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003818 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003819
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003820 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003821 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003822 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003823 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003824 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003825
Åsa Persson8c1bf952018-09-13 10:42:19 +02003826 int64_t timestamp_ms = kFrameIntervalMs;
3827 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003828 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003829 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003830 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3831 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3832
3833 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003834 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003835 timestamp_ms += kFrameIntervalMs;
3836 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3837 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003838 EXPECT_THAT(source.sink_wants(),
3839 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003840 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3841 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3842
3843 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003844 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003845 timestamp_ms += kFrameIntervalMs;
3846 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003847 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003848 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003849 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3850 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3851
3852 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003853 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003854 timestamp_ms += kFrameIntervalMs;
3855 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3856 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003857 EXPECT_THAT(source.sink_wants(),
3858 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003859 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3860 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3861
3862 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003863 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003864 timestamp_ms += kFrameIntervalMs;
3865 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003866 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003867 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003868 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3869 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3870
mflodmancc3d4422017-08-03 08:27:51 -07003871 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003872}
3873
mflodmancc3d4422017-08-03 08:27:51 -07003874TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003875 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3876 const int kWidth = 1280;
3877 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003878 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003879 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003880
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003881 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003882 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003883 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003884 video_stream_encoder_->SetSource(&source,
3885 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003886
Åsa Persson8c1bf952018-09-13 10:42:19 +02003887 int64_t timestamp_ms = kFrameIntervalMs;
3888 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003889 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003890 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003891 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3892 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3893
3894 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003895 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003896 timestamp_ms += kFrameIntervalMs;
3897 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3898 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003899 EXPECT_THAT(source.sink_wants(),
3900 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003901 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3902 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3903
3904 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003905 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003906 timestamp_ms += kFrameIntervalMs;
3907 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003908 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003909 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003910 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3911 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3912
3913 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003914 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003915 timestamp_ms += kFrameIntervalMs;
3916 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3917 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003918 EXPECT_THAT(source.sink_wants(),
3919 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003920 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3921 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3922
3923 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003924 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003925 timestamp_ms += kFrameIntervalMs;
3926 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003927 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003928 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003929 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3930 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3931
mflodmancc3d4422017-08-03 08:27:51 -07003932 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003933}
3934
Sergey Silkin41c650b2019-10-14 13:12:19 +02003935TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
3936 fake_encoder_.SetResolutionBitrateLimits(
3937 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3938
Henrik Boström381d1092020-05-12 18:49:07 +02003939 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003940 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3941 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3942 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3943 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003944
3945 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003946 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003947 source.set_adaptation_enabled(true);
3948 video_stream_encoder_->SetSource(
3949 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3950
3951 // Insert 720p frame.
3952 int64_t timestamp_ms = kFrameIntervalMs;
3953 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3954 WaitForEncodedFrame(1280, 720);
3955
3956 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02003957 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003958 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3959 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3960 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3961 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003962 video_stream_encoder_->TriggerQualityLow();
3963
3964 // Insert 720p frame. It should be downscaled and encoded.
3965 timestamp_ms += kFrameIntervalMs;
3966 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
3967 WaitForEncodedFrame(960, 540);
3968
3969 // Trigger adapt up. Higher resolution should not be requested duo to lack
3970 // of bitrate.
3971 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003972 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02003973
3974 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02003975 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003976 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3977 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
3978 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
3979 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003980
3981 // Trigger adapt up. Higher resolution should be requested.
3982 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003983 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02003984
3985 video_stream_encoder_->Stop();
3986}
3987
3988TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
3989 fake_encoder_.SetResolutionBitrateLimits(
3990 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
3991
3992 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02003993 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01003994 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3995 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
3996 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
3997 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02003998
3999 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004000 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004001 source.set_adaptation_enabled(true);
4002 video_stream_encoder_->SetSource(
4003 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4004
4005 // Insert 720p frame. It should be dropped and lower resolution should be
4006 // requested.
4007 int64_t timestamp_ms = kFrameIntervalMs;
4008 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4009 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004010 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004011
4012 // Insert 720p frame. It should be downscaled and encoded.
4013 timestamp_ms += kFrameIntervalMs;
4014 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4015 WaitForEncodedFrame(960, 540);
4016
4017 video_stream_encoder_->Stop();
4018}
4019
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004020class BalancedDegradationTest : public VideoStreamEncoderTest {
4021 protected:
4022 void SetupTest() {
4023 // Reset encoder for field trials to take effect.
4024 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02004025 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004026
4027 // Enable BALANCED preference.
4028 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02004029 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4030 }
4031
Asa Persson606d3cb2021-10-04 10:07:11 +02004032 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02004033 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004034 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004035 }
4036
Åsa Persson45b176f2019-09-30 11:19:05 +02004037 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004038 timestamp_ms_ += kFrameIntervalMs;
4039 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02004040 }
4041
4042 void InsertFrameAndWaitForEncoded() {
4043 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004044 sink_.WaitForEncodedFrame(timestamp_ms_);
4045 }
4046
4047 const int kWidth = 640; // pixels:640x360=230400
4048 const int kHeight = 360;
4049 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4050 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004051 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004052};
4053
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004054TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004055 test::ScopedFieldTrials field_trials(
4056 "WebRTC-Video-BalancedDegradationSettings/"
4057 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4058 SetupTest();
4059
4060 // Force input frame rate.
4061 const int kInputFps = 24;
4062 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4063 stats.input_frame_rate = kInputFps;
4064 stats_proxy_->SetMockStats(stats);
4065
Åsa Persson45b176f2019-09-30 11:19:05 +02004066 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004067 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004068
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004069 // Trigger adapt down, expect scaled down framerate and resolution,
4070 // since Fps diff (input-requested:0) < threshold.
4071 video_stream_encoder_->TriggerQualityLow();
4072 EXPECT_THAT(source_.sink_wants(),
4073 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004074
4075 video_stream_encoder_->Stop();
4076}
4077
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004078TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004079 test::ScopedFieldTrials field_trials(
4080 "WebRTC-Video-BalancedDegradationSettings/"
4081 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4082 SetupTest();
4083
4084 // Force input frame rate.
4085 const int kInputFps = 25;
4086 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4087 stats.input_frame_rate = kInputFps;
4088 stats_proxy_->SetMockStats(stats);
4089
Åsa Persson45b176f2019-09-30 11:19:05 +02004090 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004091 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004092
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004093 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4094 // Fps diff (input-requested:1) == threshold.
4095 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004096 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004097
4098 video_stream_encoder_->Stop();
4099}
4100
4101TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
4102 test::ScopedFieldTrials field_trials(
4103 "WebRTC-Video-BalancedDegradationSettings/"
4104 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4105 SetupTest();
4106
4107 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4108
Åsa Persson45b176f2019-09-30 11:19:05 +02004109 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004110 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004111
4112 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4113 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004114 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004115
4116 video_stream_encoder_->Stop();
4117}
4118
Åsa Perssonccfb3402019-09-25 15:13:04 +02004119TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004120 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02004121 "WebRTC-Video-BalancedDegradationSettings/"
4122 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004123 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004124
Asa Persson606d3cb2021-10-04 10:07:11 +02004125 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4126 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4127 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004128
Åsa Persson45b176f2019-09-30 11:19:05 +02004129 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004130 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004131 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4132
4133 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4134 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004135 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004136 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004137 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4138
4139 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4140 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004141 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004142 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004143 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4144
Åsa Persson30ab0152019-08-27 12:22:33 +02004145 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4146 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004147 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004148 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004149 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004150 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4151
4152 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004153 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004154 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004155 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004156
Åsa Persson30ab0152019-08-27 12:22:33 +02004157 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004158 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004159 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004160 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004161 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004162 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4163
4164 video_stream_encoder_->Stop();
4165}
4166
Åsa Perssonccfb3402019-09-25 15:13:04 +02004167TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004168 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
4169 test::ScopedFieldTrials field_trials(
4170 "WebRTC-Video-BalancedDegradationSettings/"
4171 "pixels:57600|129600|230400,fps:7|24|24/");
4172 SetupTest();
Asa Persson606d3cb2021-10-04 10:07:11 +02004173 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 11:19:05 +02004174
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004175 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004176
4177 // Insert frame, expect scaled down:
4178 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4179 InsertFrame();
4180 EXPECT_FALSE(WaitForFrame(1000));
4181 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4182 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4183
4184 // Insert frame, expect scaled down:
4185 // resolution (320x180@24fps).
4186 InsertFrame();
4187 EXPECT_FALSE(WaitForFrame(1000));
4188 EXPECT_LT(source_.sink_wants().max_pixel_count,
4189 source_.last_wants().max_pixel_count);
4190 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4191
4192 // Frame should not be dropped (min pixels per frame reached).
4193 InsertFrameAndWaitForEncoded();
4194
4195 video_stream_encoder_->Stop();
4196}
4197
4198TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004199 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004200 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004201 "WebRTC-Video-BalancedDegradationSettings/"
4202 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004203 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004204
Asa Persson606d3cb2021-10-04 10:07:11 +02004205 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4206 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4207 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004208
Åsa Persson45b176f2019-09-30 11:19:05 +02004209 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004210 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004211 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4212
4213 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4214 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004215 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004216 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004217 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4218
4219 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4220 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004221 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004222 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004223 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4224
4225 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4226 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004227 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004228 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004229 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4230
Åsa Persson30ab0152019-08-27 12:22:33 +02004231 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4232 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004233 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004234 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004235 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4236
4237 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4238 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004239 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004240 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4241
4242 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004243 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004244 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004245 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004246 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004247 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4248
4249 video_stream_encoder_->Stop();
4250}
4251
Åsa Perssonccfb3402019-09-25 15:13:04 +02004252TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004253 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004254 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004255 "WebRTC-Video-BalancedDegradationSettings/"
4256 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004257 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004258
Asa Persson606d3cb2021-10-04 10:07:11 +02004259 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4260 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4261 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4262 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4263 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004264
Åsa Persson45b176f2019-09-30 11:19:05 +02004265 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004266 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004267 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4268
4269 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4270 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004271 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004272 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004273 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4274
4275 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4276 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004277 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004278 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004279 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4280
4281 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4282 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004283 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004284 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004285 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4286
4287 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4288 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004289 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004290 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4291
4292 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004293 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004294 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004295 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004296 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004297 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4298
4299 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004300 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004301 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004302 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004303 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4304
4305 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004306 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004307 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004308 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004309 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004310 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4311
Åsa Persson1b247f12019-08-14 17:26:39 +02004312 video_stream_encoder_->Stop();
4313}
4314
mflodmancc3d4422017-08-03 08:27:51 -07004315TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004316 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4317 const int kWidth = 1280;
4318 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004319 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004320 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004321
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004322 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004323 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004324 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004325 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004326 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004327
Åsa Persson8c1bf952018-09-13 10:42:19 +02004328 int64_t timestamp_ms = kFrameIntervalMs;
4329 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004330 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004331 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004332 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4333 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4334 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4335 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4336
4337 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004338 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004339 timestamp_ms += kFrameIntervalMs;
4340 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4341 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004342 EXPECT_THAT(source.sink_wants(),
4343 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004344 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4345 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4346 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4347 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4348
4349 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004350 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004351 timestamp_ms += kFrameIntervalMs;
4352 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4353 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004354 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004355 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4356 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4357 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4358 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4359
Jonathan Yubc771b72017-12-08 17:04:29 -08004360 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004361 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004362 timestamp_ms += kFrameIntervalMs;
4363 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4364 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004365 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004366 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4367 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004368 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004369 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4370
Jonathan Yubc771b72017-12-08 17:04:29 -08004371 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004372 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004373 timestamp_ms += kFrameIntervalMs;
4374 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4375 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004376 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004377 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004378 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4379 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4380 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4381 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4382
Jonathan Yubc771b72017-12-08 17:04:29 -08004383 // Trigger quality adapt down, expect no change (min resolution reached).
4384 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004385 timestamp_ms += kFrameIntervalMs;
4386 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4387 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004388 EXPECT_THAT(source.sink_wants(), FpsMax());
4389 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004390 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4391 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4392 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4393 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4394
Evan Shrubsole64469032020-06-11 10:45:29 +02004395 // Trigger quality adapt up, expect upscaled resolution (480x270).
4396 video_stream_encoder_->TriggerQualityHigh();
4397 timestamp_ms += kFrameIntervalMs;
4398 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4399 WaitForEncodedFrame(timestamp_ms);
4400 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4401 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4402 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4403 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4404 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4405
4406 // Trigger quality and cpu adapt up since both are most limited, expect
4407 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004408 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004409 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004410 timestamp_ms += kFrameIntervalMs;
4411 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4412 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004413 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004414 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4415 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4416 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004417 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004418
Evan Shrubsole64469032020-06-11 10:45:29 +02004419 // Trigger quality and cpu adapt up since both are most limited, expect
4420 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004421 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004422 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004423 timestamp_ms += kFrameIntervalMs;
4424 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4425 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004426 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004427 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004428 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004429 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004430 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4431 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004432
Evan Shrubsole64469032020-06-11 10:45:29 +02004433 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4434 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004435 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004436 timestamp_ms += kFrameIntervalMs;
4437 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4438 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004439 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004440 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4441 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004442 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004443 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004444
4445 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004446 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004447 timestamp_ms += kFrameIntervalMs;
4448 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004449 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004450 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004451 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004452 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4453 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004454 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004455 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004456
mflodmancc3d4422017-08-03 08:27:51 -07004457 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004458}
4459
mflodmancc3d4422017-08-03 08:27:51 -07004460TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004461 const int kWidth = 640;
4462 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004463
Henrik Boström381d1092020-05-12 18:49:07 +02004464 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004465 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004466
perkj803d97f2016-11-01 11:45:46 -07004467 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004468 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004469 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004470 }
4471
mflodmancc3d4422017-08-03 08:27:51 -07004472 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004473 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004474 video_source_.IncomingCapturedFrame(CreateFrame(
4475 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004476 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004477 }
4478
mflodmancc3d4422017-08-03 08:27:51 -07004479 video_stream_encoder_->Stop();
4480 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004481 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004482
Ying Wangef3998f2019-12-09 13:06:53 +01004483 EXPECT_METRIC_EQ(
4484 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4485 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004486 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4487}
4488
mflodmancc3d4422017-08-03 08:27:51 -07004489TEST_F(VideoStreamEncoderTest,
4490 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004491 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004492 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004493 const int kWidth = 640;
4494 const int kHeight = 360;
4495
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004496 video_stream_encoder_->SetSource(&video_source_,
4497 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004498
4499 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4500 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004501 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004502 }
4503
mflodmancc3d4422017-08-03 08:27:51 -07004504 video_stream_encoder_->Stop();
4505 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004506 stats_proxy_.reset();
4507
4508 EXPECT_EQ(0,
4509 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4510}
4511
Per Kjellanderdcef6412020-10-07 15:09:05 +02004512TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4513 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004514 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004515 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004516
4517 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004518 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 10:07:11 +02004519 SimulcastRateAllocator(fake_encoder_.config())
4520 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004521 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004522
Henrik Boström381d1092020-05-12 18:49:07 +02004523 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004524 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004525
sprang57c2fff2017-01-16 06:24:02 -08004526 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004527 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4528 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004529 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4530 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4531
Erik Språngd7329ca2019-02-21 21:19:53 +01004532 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004533 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004534 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004535
Per Kjellanderdcef6412020-10-07 15:09:05 +02004536 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004537 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004538 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4539 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004540 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004541 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004542
Per Kjellanderdcef6412020-10-07 15:09:05 +02004543 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004544 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004545 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004546 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004547 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4548 WaitForEncodedFrame(CurrentTimeMs());
4549 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004550 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004551 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004552
mflodmancc3d4422017-08-03 08:27:51 -07004553 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004554}
4555
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004556TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004557 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004558 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004559 kVideoLayersAllocation);
4560
4561 const int kDefaultFps = 30;
4562
4563 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004564 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004565
4566 video_source_.IncomingCapturedFrame(
4567 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4568 WaitForEncodedFrame(CurrentTimeMs());
4569 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4570 VideoLayersAllocation last_layer_allocation =
4571 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004572 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004573 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4574
4575 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004576 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004577 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 10:07:11 +02004578 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 17:53:22 +02004579 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4580
Erik Språng9d69cbe2020-10-22 17:44:42 +02004581 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004582 int number_of_layers_allocation = 1;
4583 const int64_t start_time_ms = CurrentTimeMs();
4584 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4585 video_source_.IncomingCapturedFrame(
4586 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4587 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004588 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4589 number_of_layers_allocation = sink_.number_of_layers_allocations();
4590 VideoLayersAllocation new_allocation =
4591 sink_.GetLastVideoLayersAllocation();
4592 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4593 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4594 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4595 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4596 .target_bitrate_per_temporal_layer,
4597 last_layer_allocation.active_spatial_layers[0]
4598 .target_bitrate_per_temporal_layer);
4599 last_layer_allocation = new_allocation;
4600 }
4601 }
4602 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4603 video_stream_encoder_->Stop();
4604}
4605
4606TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004607 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004608 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4609 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4610 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004611 VideoEncoderConfig video_encoder_config;
4612 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4613 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004614 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004615 video_encoder_config.content_type =
4616 VideoEncoderConfig::ContentType::kRealtimeVideo;
4617 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004618 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004619 VideoEncoder::GetDefaultVp8Settings());
4620 for (auto& layer : video_encoder_config.simulcast_layers) {
4621 layer.num_temporal_layers = 2;
4622 }
4623 // Simulcast layers are used for enabling/disabling streams.
4624 video_encoder_config.simulcast_layers[0].active = true;
4625 video_encoder_config.simulcast_layers[1].active = false;
4626 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004627 ConfigureEncoder(std::move(video_encoder_config),
4628 VideoStreamEncoder::BitrateAllocationCallbackType::
4629 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004630
4631 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004632 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004633
4634 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4635 WaitForEncodedFrame(CurrentTimeMs());
4636 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4637 VideoLayersAllocation last_layer_allocation =
4638 sink_.GetLastVideoLayersAllocation();
4639
4640 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4641 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4642 .target_bitrate_per_temporal_layer,
4643 SizeIs(2));
4644 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4645 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4646 video_stream_encoder_->Stop();
4647}
4648
4649TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004650 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004651 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4652 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4653 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004654 VideoEncoderConfig video_encoder_config;
4655 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4656 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004657 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004658 video_encoder_config.content_type =
4659 VideoEncoderConfig::ContentType::kRealtimeVideo;
4660 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004661 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004662 VideoEncoder::GetDefaultVp8Settings());
4663 for (auto& layer : video_encoder_config.simulcast_layers) {
4664 layer.num_temporal_layers = 2;
4665 }
4666 // Simulcast layers are used for enabling/disabling streams.
4667 video_encoder_config.simulcast_layers[0].active = true;
4668 video_encoder_config.simulcast_layers[1].active = false;
4669 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004670 ConfigureEncoder(std::move(video_encoder_config),
4671 VideoStreamEncoder::BitrateAllocationCallbackType::
4672 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004673
4674 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004675 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004676
4677 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4678 WaitForEncodedFrame(CurrentTimeMs());
4679 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4680 VideoLayersAllocation last_layer_allocation =
4681 sink_.GetLastVideoLayersAllocation();
4682
4683 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4684 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4685 .target_bitrate_per_temporal_layer,
4686 SizeIs(2));
4687 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4688
4689 video_stream_encoder_->Stop();
4690}
4691
4692TEST_F(VideoStreamEncoderTest,
4693 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4694 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4695 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004696 VideoEncoderConfig video_encoder_config;
4697 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4698 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004699 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004700 video_encoder_config.content_type =
4701 VideoEncoderConfig::ContentType::kRealtimeVideo;
4702 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4703 vp9_settings.numberOfSpatialLayers = 2;
4704 vp9_settings.numberOfTemporalLayers = 2;
4705 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4706 vp9_settings.automaticResizeOn = false;
4707 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004708 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004709 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004710 ConfigureEncoder(std::move(video_encoder_config),
4711 VideoStreamEncoder::BitrateAllocationCallbackType::
4712 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004713
4714 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004715 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004716
4717 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4718 WaitForEncodedFrame(CurrentTimeMs());
4719 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4720 VideoLayersAllocation last_layer_allocation =
4721 sink_.GetLastVideoLayersAllocation();
4722
4723 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4724 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4725 .target_bitrate_per_temporal_layer,
4726 SizeIs(2));
4727 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4728 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4729 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4730 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4731 .target_bitrate_per_temporal_layer,
4732 SizeIs(2));
4733 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4734 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4735 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4736
4737 // Since full SVC is used, expect the top layer to utilize the full target
4738 // rate.
4739 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4740 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004741 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004742 video_stream_encoder_->Stop();
4743}
4744
4745TEST_F(VideoStreamEncoderTest,
4746 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4747 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4748 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004749 VideoEncoderConfig video_encoder_config;
4750 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4751 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004752 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004753 video_encoder_config.content_type =
4754 VideoEncoderConfig::ContentType::kRealtimeVideo;
4755 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4756 vp9_settings.numberOfSpatialLayers = 2;
4757 vp9_settings.numberOfTemporalLayers = 2;
4758 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4759 vp9_settings.automaticResizeOn = false;
4760 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004761 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004762 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004763 ConfigureEncoder(std::move(video_encoder_config),
4764 VideoStreamEncoder::BitrateAllocationCallbackType::
4765 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004766
4767 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004768 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004769
4770 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4771 WaitForEncodedFrame(CurrentTimeMs());
4772 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4773 VideoLayersAllocation last_layer_allocation =
4774 sink_.GetLastVideoLayersAllocation();
4775
4776 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4777 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4778 .target_bitrate_per_temporal_layer,
4779 SizeIs(1));
4780 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4781 .target_bitrate_per_temporal_layer,
4782 SizeIs(1));
4783 // Since full SVC is used, expect the top layer to utilize the full target
4784 // rate.
4785 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4786 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004787 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004788 video_stream_encoder_->Stop();
4789}
4790
4791TEST_F(VideoStreamEncoderTest,
4792 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4793 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4794 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004795 VideoEncoderConfig video_encoder_config;
4796 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4797 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004798 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004799 video_encoder_config.content_type =
4800 VideoEncoderConfig::ContentType::kRealtimeVideo;
4801 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4802 vp9_settings.numberOfSpatialLayers = 2;
4803 vp9_settings.numberOfTemporalLayers = 2;
4804 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4805 vp9_settings.automaticResizeOn = false;
4806 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004807 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004808 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004809 ConfigureEncoder(std::move(video_encoder_config),
4810 VideoStreamEncoder::BitrateAllocationCallbackType::
4811 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004812
4813 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004814 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004815
4816 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4817 WaitForEncodedFrame(CurrentTimeMs());
4818 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4819 VideoLayersAllocation last_layer_allocation =
4820 sink_.GetLastVideoLayersAllocation();
4821
4822 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4823 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4824 .target_bitrate_per_temporal_layer,
4825 SizeIs(2));
4826 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4827 .target_bitrate_per_temporal_layer,
4828 SizeIs(2));
4829 // Since KSVC is, spatial layers are independend except on key frames.
4830 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4831 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004832 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004833 video_stream_encoder_->Stop();
4834}
4835
4836TEST_F(VideoStreamEncoderTest,
4837 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4838 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4839 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4840 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004841 VideoEncoderConfig video_encoder_config;
4842 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4843 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004844 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004845 video_encoder_config.content_type =
4846 VideoEncoderConfig::ContentType::kRealtimeVideo;
4847 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4848 vp9_settings.numberOfSpatialLayers = 3;
4849 vp9_settings.numberOfTemporalLayers = 2;
4850 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4851 vp9_settings.automaticResizeOn = false;
4852 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004853 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004854 vp9_settings);
4855 // Simulcast layers are used for enabling/disabling streams.
4856 video_encoder_config.simulcast_layers.resize(3);
4857 video_encoder_config.simulcast_layers[0].active = false;
4858 video_encoder_config.simulcast_layers[1].active = true;
4859 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004860 ConfigureEncoder(std::move(video_encoder_config),
4861 VideoStreamEncoder::BitrateAllocationCallbackType::
4862 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004863
4864 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004865 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004866
4867 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4868 WaitForEncodedFrame(CurrentTimeMs());
4869 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4870 VideoLayersAllocation last_layer_allocation =
4871 sink_.GetLastVideoLayersAllocation();
4872
4873 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4874 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4875 .target_bitrate_per_temporal_layer,
4876 SizeIs(2));
4877 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4878 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4879
4880 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4881 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4882 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4883 .target_bitrate_per_temporal_layer,
4884 SizeIs(2));
4885 // Since full SVC is used, expect the top layer to utilize the full target
4886 // rate.
4887 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4888 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004889 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004890 video_stream_encoder_->Stop();
4891}
4892
4893TEST_F(VideoStreamEncoderTest,
4894 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4895 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4896 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4897 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004898 VideoEncoderConfig video_encoder_config;
4899 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4900 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004901 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004902 video_encoder_config.content_type =
4903 VideoEncoderConfig::ContentType::kRealtimeVideo;
4904 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4905 vp9_settings.numberOfSpatialLayers = 3;
4906 vp9_settings.numberOfTemporalLayers = 2;
4907 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4908 vp9_settings.automaticResizeOn = false;
4909 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004910 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004911 vp9_settings);
4912 // Simulcast layers are used for enabling/disabling streams.
4913 video_encoder_config.simulcast_layers.resize(3);
4914 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004915 ConfigureEncoder(std::move(video_encoder_config),
4916 VideoStreamEncoder::BitrateAllocationCallbackType::
4917 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004918
4919 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004920 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004921
4922 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4923 WaitForEncodedFrame(CurrentTimeMs());
4924 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4925 VideoLayersAllocation last_layer_allocation =
4926 sink_.GetLastVideoLayersAllocation();
4927
4928 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4929 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4930 .target_bitrate_per_temporal_layer,
4931 SizeIs(2));
4932 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
4933 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4934
4935 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
4936 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4937 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4938 .target_bitrate_per_temporal_layer,
4939 SizeIs(2));
4940 video_stream_encoder_->Stop();
4941}
4942
4943TEST_F(VideoStreamEncoderTest,
4944 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
4945 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4946 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4947 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004948 VideoEncoderConfig video_encoder_config;
4949 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4950 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004951 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004952 video_encoder_config.content_type =
4953 VideoEncoderConfig::ContentType::kRealtimeVideo;
4954 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4955 vp9_settings.numberOfSpatialLayers = 3;
4956 vp9_settings.numberOfTemporalLayers = 2;
4957 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4958 vp9_settings.automaticResizeOn = false;
4959 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004960 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004961 vp9_settings);
4962 // Simulcast layers are used for enabling/disabling streams.
4963 video_encoder_config.simulcast_layers.resize(3);
4964 video_encoder_config.simulcast_layers[0].active = false;
4965 video_encoder_config.simulcast_layers[1].active = false;
4966 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004967 ConfigureEncoder(std::move(video_encoder_config),
4968 VideoStreamEncoder::BitrateAllocationCallbackType::
4969 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004970
4971 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004972 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004973
4974 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4975 WaitForEncodedFrame(CurrentTimeMs());
4976 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4977 VideoLayersAllocation last_layer_allocation =
4978 sink_.GetLastVideoLayersAllocation();
4979
4980 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4981 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4982 .target_bitrate_per_temporal_layer,
4983 SizeIs(2));
4984 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
4985 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4986 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
4987 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004988 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004989 video_stream_encoder_->Stop();
4990}
4991
4992TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
4993 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004994 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004995 kVideoLayersAllocation);
4996 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004997 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004998
4999 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5000 WaitForEncodedFrame(CurrentTimeMs());
5001 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5002 VideoLayersAllocation last_layer_allocation =
5003 sink_.GetLastVideoLayersAllocation();
5004
5005 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5006 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5007 .target_bitrate_per_temporal_layer,
5008 SizeIs(1));
5009 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5010 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005011 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005012 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5013 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5014 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5015 video_stream_encoder_->Stop();
5016}
5017
5018TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02005019 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5020 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005021 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02005022 kVideoLayersAllocation);
5023
5024 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005025 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005026
5027 video_source_.IncomingCapturedFrame(
5028 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5029 WaitForEncodedFrame(CurrentTimeMs());
5030 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5031 VideoLayersAllocation last_layer_allocation =
5032 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02005033 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02005034 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5035 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5036 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005037 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 17:53:22 +02005038
5039 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005040 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5041 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005042 video_source_.IncomingCapturedFrame(
5043 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5044 WaitForEncodedFrame(CurrentTimeMs());
5045
5046 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5047 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5048 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5049 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5050 .target_bitrate_per_temporal_layer[0],
5051 DataRate::Zero());
5052
5053 video_stream_encoder_->Stop();
5054}
5055
Per Kjellander4190ce92020-12-15 17:24:55 +01005056TEST_F(VideoStreamEncoderTest,
5057 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5058 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005059 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01005060 kVideoLayersAllocation);
5061
5062 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005063 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5064 0, 0, 0);
Per Kjellander4190ce92020-12-15 17:24:55 +01005065
5066 video_source_.IncomingCapturedFrame(
5067 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5068 WaitForEncodedFrame(CurrentTimeMs());
5069 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5070 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5071 SizeIs(2));
5072 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5073 codec_width_);
5074 EXPECT_EQ(
5075 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5076 codec_height_);
5077
5078 video_source_.IncomingCapturedFrame(
5079 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5080 WaitForEncodedFrame(CurrentTimeMs());
5081 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5082 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5083 SizeIs(2));
5084 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5085 codec_width_ / 2);
5086 EXPECT_EQ(
5087 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5088 codec_height_ / 2);
5089
5090 video_stream_encoder_->Stop();
5091}
5092
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005093TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5094 // 2 TLs configured, temporal layers supported by encoder.
5095 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005096 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005097 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005098 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005099 fake_encoder_.SetTemporalLayersSupported(0, true);
5100
5101 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 10:07:11 +02005102 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005103 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005104 kNumTemporalLayers, /*temporal_id*/ 0,
5105 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005106 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005107 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005108 kNumTemporalLayers, /*temporal_id*/ 1,
5109 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005110 VideoBitrateAllocation expected_bitrate;
5111 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5112 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5113
5114 VerifyAllocatedBitrate(expected_bitrate);
5115 video_stream_encoder_->Stop();
5116}
5117
5118TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5119 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005120 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005121 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005122 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005123 fake_encoder_.SetTemporalLayersSupported(0, false);
5124
5125 // Temporal layers not supported by the encoder.
5126 // Total bitrate should be at ti:0.
5127 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005128 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005129
5130 VerifyAllocatedBitrate(expected_bitrate);
5131 video_stream_encoder_->Stop();
5132}
5133
5134TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 15:09:05 +02005135 webrtc::test::ScopedFieldTrials field_trials(
5136 "WebRTC-Video-QualityScalerSettings/"
5137 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5138 // Reset encoder for field trials to take effect.
5139 ConfigureEncoder(video_encoder_config_.Copy());
5140
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005141 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005142 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005143 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005144 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005145 fake_encoder_.SetTemporalLayersSupported(0, true);
5146 fake_encoder_.SetTemporalLayersSupported(1, false);
5147
5148 const int kS0Bps = 150000;
5149 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005150 kS0Bps *
5151 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5152 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005153 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005154 kS0Bps *
5155 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5156 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005157 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005158 // Temporal layers not supported by si:1.
5159 VideoBitrateAllocation expected_bitrate;
5160 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5161 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5162 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5163
5164 VerifyAllocatedBitrate(expected_bitrate);
5165 video_stream_encoder_->Stop();
5166}
5167
Niels Möller7dc26b72017-12-06 10:27:48 +01005168TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5169 const int kFrameWidth = 1280;
5170 const int kFrameHeight = 720;
5171 const int kFramerate = 24;
5172
Henrik Boström381d1092020-05-12 18:49:07 +02005173 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005174 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005175 test::FrameForwarder source;
5176 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005177 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005178
5179 // Insert a single frame, triggering initial configuration.
5180 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5181 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5182
5183 EXPECT_EQ(
5184 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5185 kDefaultFramerate);
5186
5187 // Trigger reconfigure encoder (without resetting the entire instance).
5188 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005189 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5190 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005191 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005192 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005193 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005194 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5195
5196 // Detector should be updated with fps limit from codec config.
5197 EXPECT_EQ(
5198 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5199 kFramerate);
5200
5201 // Trigger overuse, max framerate should be reduced.
5202 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5203 stats.input_frame_rate = kFramerate;
5204 stats_proxy_->SetMockStats(stats);
5205 video_stream_encoder_->TriggerCpuOveruse();
5206 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5207 int adapted_framerate =
5208 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5209 EXPECT_LT(adapted_framerate, kFramerate);
5210
5211 // Trigger underuse, max framerate should go back to codec configured fps.
5212 // Set extra low fps, to make sure it's actually reset, not just incremented.
5213 stats = stats_proxy_->GetStats();
5214 stats.input_frame_rate = adapted_framerate / 2;
5215 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005216 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005217 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5218 EXPECT_EQ(
5219 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5220 kFramerate);
5221
5222 video_stream_encoder_->Stop();
5223}
5224
5225TEST_F(VideoStreamEncoderTest,
5226 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5227 const int kFrameWidth = 1280;
5228 const int kFrameHeight = 720;
5229 const int kLowFramerate = 15;
5230 const int kHighFramerate = 25;
5231
Henrik Boström381d1092020-05-12 18:49:07 +02005232 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005233 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005234 test::FrameForwarder source;
5235 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005236 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005237
5238 // Trigger initial configuration.
5239 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005240 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5241 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005242 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005243 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005244 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005245 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005246 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5247
5248 EXPECT_EQ(
5249 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5250 kLowFramerate);
5251
5252 // Trigger overuse, max framerate should be reduced.
5253 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5254 stats.input_frame_rate = kLowFramerate;
5255 stats_proxy_->SetMockStats(stats);
5256 video_stream_encoder_->TriggerCpuOveruse();
5257 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5258 int adapted_framerate =
5259 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5260 EXPECT_LT(adapted_framerate, kLowFramerate);
5261
5262 // Reconfigure the encoder with a new (higher max framerate), max fps should
5263 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005264 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005265 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5266 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005267 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005268 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5269
5270 EXPECT_EQ(
5271 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5272 adapted_framerate);
5273
5274 // Trigger underuse, max framerate should go back to codec configured fps.
5275 stats = stats_proxy_->GetStats();
5276 stats.input_frame_rate = adapted_framerate;
5277 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005278 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005279 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5280 EXPECT_EQ(
5281 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5282 kHighFramerate);
5283
5284 video_stream_encoder_->Stop();
5285}
5286
mflodmancc3d4422017-08-03 08:27:51 -07005287TEST_F(VideoStreamEncoderTest,
5288 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005289 const int kFrameWidth = 1280;
5290 const int kFrameHeight = 720;
5291 const int kFramerate = 24;
5292
Henrik Boström381d1092020-05-12 18:49:07 +02005293 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005294 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005295 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005296 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005297 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005298
5299 // Trigger initial configuration.
5300 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005301 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5302 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005303 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 04:21:07 -07005304 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005305 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005306 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005307 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005308
Niels Möller7dc26b72017-12-06 10:27:48 +01005309 EXPECT_EQ(
5310 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5311 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005312
5313 // Trigger overuse, max framerate should be reduced.
5314 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5315 stats.input_frame_rate = kFramerate;
5316 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005317 video_stream_encoder_->TriggerCpuOveruse();
5318 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005319 int adapted_framerate =
5320 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005321 EXPECT_LT(adapted_framerate, kFramerate);
5322
5323 // Change degradation preference to not enable framerate scaling. Target
5324 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005325 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005326 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005327 EXPECT_EQ(
5328 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5329 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005330
mflodmancc3d4422017-08-03 08:27:51 -07005331 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005332}
5333
mflodmancc3d4422017-08-03 08:27:51 -07005334TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005335 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005336 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005337 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5338 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5339 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005340 const int kWidth = 640;
5341 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005342
asaperssonfab67072017-04-04 05:51:49 -07005343 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005344
5345 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005346 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005347
5348 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005349 EXPECT_TRUE_WAIT(
5350 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005351
sprangc5d62e22017-04-02 23:53:04 -07005352 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005353
asaperssonfab67072017-04-04 05:51:49 -07005354 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005355 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005356 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005357
5358 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005359 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005360
Henrik Boström2671dac2020-05-19 16:29:09 +02005361 EXPECT_TRUE_WAIT(
5362 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005363
mflodmancc3d4422017-08-03 08:27:51 -07005364 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005365}
5366
mflodmancc3d4422017-08-03 08:27:51 -07005367TEST_F(VideoStreamEncoderTest,
5368 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005369 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005370 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005371 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5372 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5373 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005374 const int kWidth = 640;
5375 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005376
5377 // We expect the n initial frames to get dropped.
5378 int i;
5379 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005380 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005381 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005382 }
5383 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005384 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005385 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005386
5387 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005388 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005389
mflodmancc3d4422017-08-03 08:27:51 -07005390 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005391}
5392
mflodmancc3d4422017-08-03 08:27:51 -07005393TEST_F(VideoStreamEncoderTest,
5394 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005395 const int kWidth = 640;
5396 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005397 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005398 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005399
5400 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005401 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005402 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005403
asaperssonfab67072017-04-04 05:51:49 -07005404 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005405 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005406 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005407
mflodmancc3d4422017-08-03 08:27:51 -07005408 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005409}
5410
mflodmancc3d4422017-08-03 08:27:51 -07005411TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005412 const int kWidth = 640;
5413 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005414 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005415
5416 VideoEncoderConfig video_encoder_config;
5417 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5418 // Make format different, to force recreation of encoder.
5419 video_encoder_config.video_format.parameters["foo"] = "foo";
5420 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005421 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005422 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005423 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005424
kthelgasonb83797b2017-02-14 11:57:25 -08005425 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005426 video_stream_encoder_->SetSource(&video_source_,
5427 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005428
asaperssonfab67072017-04-04 05:51:49 -07005429 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005430 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005431 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005432
mflodmancc3d4422017-08-03 08:27:51 -07005433 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005434 fake_encoder_.SetQualityScaling(true);
5435}
5436
Åsa Persson139f4dc2019-08-02 09:29:58 +02005437TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
5438 webrtc::test::ScopedFieldTrials field_trials(
5439 "WebRTC-Video-QualityScalerSettings/"
5440 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5441 // Reset encoder for field trials to take effect.
5442 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005443 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5444 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 09:29:58 +02005445 const int kWidth = 640;
5446 const int kHeight = 360;
5447
Henrik Boström381d1092020-05-12 18:49:07 +02005448 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005449 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005450 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5451 // Frame should not be dropped.
5452 WaitForEncodedFrame(1);
5453
Henrik Boström381d1092020-05-12 18:49:07 +02005454 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005455 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5456 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5457 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005458 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5459 // Frame should not be dropped.
5460 WaitForEncodedFrame(2);
5461
Henrik Boström381d1092020-05-12 18:49:07 +02005462 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005463 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5464 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5465 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005466 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5467 // Expect to drop this frame, the wait should time out.
5468 ExpectDroppedFrame();
5469
5470 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005471 EXPECT_TRUE_WAIT(
5472 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005473 video_stream_encoder_->Stop();
5474}
5475
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005476TEST_F(VideoStreamEncoderTest,
5477 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5478 webrtc::test::ScopedFieldTrials field_trials(
5479 "WebRTC-Video-QualityScalerSettings/"
5480 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5481 fake_encoder_.SetQualityScaling(false);
5482 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005483 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5484 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005485 const int kWidth = 640;
5486 const int kHeight = 360;
5487
5488 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005489 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005490 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5491 // Frame should not be dropped.
5492 WaitForEncodedFrame(1);
5493
5494 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5495 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5496 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5497 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5498 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5499 // Frame should not be dropped.
5500 WaitForEncodedFrame(2);
5501
5502 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5503 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5504 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5505 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5506 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5507 // Not dropped since quality scaling is disabled.
5508 WaitForEncodedFrame(3);
5509
5510 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005511 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005512 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5513
5514 video_stream_encoder_->Stop();
5515}
5516
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005517TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005518 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005519 // Set simulcast.
5520 ResetEncoder("VP8", 3, 1, 1, false);
5521 fake_encoder_.SetQualityScaling(true);
5522 const int kWidth = 1280;
5523 const int kHeight = 720;
5524 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005525 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005526 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5527 // Frame should not be dropped.
5528 WaitForEncodedFrame(1);
5529
5530 // Trigger QVGA "singlecast"
5531 // Update the config.
5532 VideoEncoderConfig video_encoder_config;
5533 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5534 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005535 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005536 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005537 "VP8", /*max qp*/ 56, /*screencast*/ false,
5538 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005539 for (auto& layer : video_encoder_config.simulcast_layers) {
5540 layer.num_temporal_layers = 1;
5541 layer.max_framerate = kDefaultFramerate;
5542 }
Asa Persson606d3cb2021-10-04 10:07:11 +02005543 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005544 video_encoder_config.content_type =
5545 VideoEncoderConfig::ContentType::kRealtimeVideo;
5546
5547 video_encoder_config.simulcast_layers[0].active = true;
5548 video_encoder_config.simulcast_layers[1].active = false;
5549 video_encoder_config.simulcast_layers[2].active = false;
5550
5551 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5552 kMaxPayloadLength);
5553 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5554
5555 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5556 // Frame should not be dropped.
5557 WaitForEncodedFrame(2);
5558
5559 // Trigger HD "singlecast"
5560 video_encoder_config.simulcast_layers[0].active = false;
5561 video_encoder_config.simulcast_layers[1].active = false;
5562 video_encoder_config.simulcast_layers[2].active = true;
5563
5564 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5565 kMaxPayloadLength);
5566 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5567
5568 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5569 // Frame should be dropped because of initial frame drop.
5570 ExpectDroppedFrame();
5571
5572 // Expect the sink_wants to specify a scaled frame.
5573 EXPECT_TRUE_WAIT(
5574 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5575 video_stream_encoder_->Stop();
5576}
5577
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005578TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005579 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005580 // Set simulcast.
5581 ResetEncoder("VP9", 1, 1, 3, false);
5582 fake_encoder_.SetQualityScaling(true);
5583 const int kWidth = 1280;
5584 const int kHeight = 720;
5585 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005586 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005587 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5588 // Frame should not be dropped.
5589 WaitForEncodedFrame(1);
5590
5591 // Trigger QVGA "singlecast"
5592 // Update the config.
5593 VideoEncoderConfig video_encoder_config;
5594 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5595 &video_encoder_config);
5596 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5597 vp9_settings.numberOfSpatialLayers = 3;
5598 // Since only one layer is active - automatic resize should be enabled.
5599 vp9_settings.automaticResizeOn = true;
5600 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005601 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005602 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005603 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005604 video_encoder_config.content_type =
5605 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 16:22:11 +02005606 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005607 // which SVC layers are active.
5608 video_encoder_config.simulcast_layers.resize(3);
5609
5610 video_encoder_config.simulcast_layers[0].active = true;
5611 video_encoder_config.simulcast_layers[1].active = false;
5612 video_encoder_config.simulcast_layers[2].active = false;
5613
5614 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5615 kMaxPayloadLength);
5616 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5617
5618 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5619 // Frame should not be dropped.
5620 WaitForEncodedFrame(2);
5621
5622 // Trigger HD "singlecast"
5623 video_encoder_config.simulcast_layers[0].active = false;
5624 video_encoder_config.simulcast_layers[1].active = false;
5625 video_encoder_config.simulcast_layers[2].active = true;
5626
5627 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5628 kMaxPayloadLength);
5629 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5630
5631 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5632 // Frame should be dropped because of initial frame drop.
5633 ExpectDroppedFrame();
5634
5635 // Expect the sink_wants to specify a scaled frame.
5636 EXPECT_TRUE_WAIT(
5637 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5638 video_stream_encoder_->Stop();
5639}
5640
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005641TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005642 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5643 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5644 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5645 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5646 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5647 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5648 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5649 fake_encoder_.SetResolutionBitrateLimits(
5650 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5651
5652 VideoEncoderConfig video_encoder_config;
5653 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5654 &video_encoder_config);
5655 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5656 vp9_settings.numberOfSpatialLayers = 3;
5657 // Since only one layer is active - automatic resize should be enabled.
5658 vp9_settings.automaticResizeOn = true;
5659 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005660 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005661 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005662 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005663 video_encoder_config.content_type =
5664 VideoEncoderConfig::ContentType::kRealtimeVideo;
5665 // Simulcast layers are used to indicate which spatial layers are active.
5666 video_encoder_config.simulcast_layers.resize(3);
5667 video_encoder_config.simulcast_layers[0].active = false;
5668 video_encoder_config.simulcast_layers[1].active = true;
5669 video_encoder_config.simulcast_layers[2].active = false;
5670
5671 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5672 kMaxPayloadLength);
5673 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5674
5675 // The encoder bitrate limits for 360p should be used.
5676 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5677 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005678 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5679 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5680 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5681 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5682 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5683 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005684 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005685 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005686 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005687 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005688
5689 // The encoder bitrate limits for 270p should be used.
5690 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5691 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005692 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5693 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5694 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5695 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5696 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5697 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005698 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005699 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005700 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005701 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005702
5703 video_stream_encoder_->Stop();
5704}
5705
5706TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005707 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5708 VideoEncoderConfig video_encoder_config;
5709 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5710 &video_encoder_config);
5711 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5712 vp9_settings.numberOfSpatialLayers = 3;
5713 // Since only one layer is active - automatic resize should be enabled.
5714 vp9_settings.automaticResizeOn = true;
5715 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005716 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005717 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005718 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005719 video_encoder_config.content_type =
5720 VideoEncoderConfig::ContentType::kRealtimeVideo;
5721 // Simulcast layers are used to indicate which spatial layers are active.
5722 video_encoder_config.simulcast_layers.resize(3);
5723 video_encoder_config.simulcast_layers[0].active = false;
5724 video_encoder_config.simulcast_layers[1].active = true;
5725 video_encoder_config.simulcast_layers[2].active = false;
5726
5727 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5728 kMaxPayloadLength);
5729 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5730
5731 // The default bitrate limits for 360p should be used.
5732 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005733 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5734 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005735 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5736 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005737 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5738 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5739 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5740 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5741 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5742 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005743 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005744 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005745 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005746 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005747
5748 // The default bitrate limits for 270p should be used.
5749 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005750 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5751 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005752 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5753 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005754 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5755 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5756 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5757 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5758 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5759 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005760 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005761 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005762 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005763 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005764
5765 video_stream_encoder_->Stop();
5766}
5767
5768TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
5769 webrtc::test::ScopedFieldTrials field_trials(
5770 "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
5771 VideoEncoderConfig video_encoder_config;
5772 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5773 &video_encoder_config);
5774 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5775 vp9_settings.numberOfSpatialLayers = 3;
5776 // Since only one layer is active - automatic resize should be enabled.
5777 vp9_settings.automaticResizeOn = true;
5778 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005779 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005780 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005781 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005782 video_encoder_config.content_type =
5783 VideoEncoderConfig::ContentType::kRealtimeVideo;
5784 // Simulcast layers are used to indicate which spatial layers are active.
5785 video_encoder_config.simulcast_layers.resize(3);
5786 video_encoder_config.simulcast_layers[0].active = false;
5787 video_encoder_config.simulcast_layers[1].active = true;
5788 video_encoder_config.simulcast_layers[2].active = false;
5789
5790 // Reset encoder for field trials to take effect.
5791 ConfigureEncoder(video_encoder_config.Copy());
5792
5793 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5794 kMaxPayloadLength);
5795 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5796
5797 // The default bitrate limits for 360p should not be used.
5798 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005799 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5800 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005801 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5802 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005803 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5804 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5805 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5806 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5807 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5808 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005809 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005810 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005811
5812 video_stream_encoder_->Stop();
5813}
5814
5815TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5816 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5817 /*num_spatial_layers=*/1, /*screenshare=*/false);
5818
5819 // The default singlecast bitrate limits for 720p should not be used.
5820 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005821 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5822 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005823 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5824 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005825 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5826 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5827 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5828 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5829 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5830 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005831 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005832 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005833
5834 video_stream_encoder_->Stop();
5835}
5836
5837TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005838 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5839 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5840 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5841 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5842 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5843 fake_encoder_.SetResolutionBitrateLimits(
5844 {kEncoderLimits180p, kEncoderLimits720p});
5845
5846 VideoEncoderConfig video_encoder_config;
5847 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5848 &video_encoder_config);
5849 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5850 vp9_settings.numberOfSpatialLayers = 3;
5851 // Since only one layer is active - automatic resize should be enabled.
5852 vp9_settings.automaticResizeOn = true;
5853 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005854 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005855 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005856 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005857 video_encoder_config.content_type =
5858 VideoEncoderConfig::ContentType::kRealtimeVideo;
5859 // Simulcast layers are used to indicate which spatial layers are active.
5860 video_encoder_config.simulcast_layers.resize(3);
5861 video_encoder_config.simulcast_layers[0].active = true;
5862 video_encoder_config.simulcast_layers[1].active = false;
5863 video_encoder_config.simulcast_layers[2].active = false;
5864
5865 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5866 kMaxPayloadLength);
5867 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5868
5869 // Limits not applied on lowest stream, limits for 180p should not be used.
5870 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5871 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005872 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5873 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5874 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5875 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5876 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5877 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005878 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005879 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005880 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005881 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005882
5883 video_stream_encoder_->Stop();
5884}
5885
5886TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005887 InitialFrameDropActivatesWhenResolutionIncreases) {
5888 const int kWidth = 640;
5889 const int kHeight = 360;
5890
5891 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005892 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005893 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5894 // Frame should not be dropped.
5895 WaitForEncodedFrame(1);
5896
5897 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005898 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005899 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5900 // Frame should not be dropped, bitrate not too low for frame.
5901 WaitForEncodedFrame(2);
5902
5903 // Incoming resolution increases.
5904 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5905 // Expect to drop this frame, bitrate too low for frame.
5906 ExpectDroppedFrame();
5907
5908 // Expect the sink_wants to specify a scaled frame.
5909 EXPECT_TRUE_WAIT(
5910 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5911 video_stream_encoder_->Stop();
5912}
5913
5914TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
5915 const int kWidth = 640;
5916 const int kHeight = 360;
5917 // So that quality scaling doesn't happen by itself.
5918 fake_encoder_.SetQp(kQpHigh);
5919
5920 AdaptingFrameForwarder source(&time_controller_);
5921 source.set_adaptation_enabled(true);
5922 video_stream_encoder_->SetSource(
5923 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
5924
5925 int timestamp = 1;
5926
5927 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005928 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005929 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5930 WaitForEncodedFrame(timestamp);
5931 timestamp += 9000;
5932 // Long pause to disable all first BWE drop logic.
5933 AdvanceTime(TimeDelta::Millis(1000));
5934
5935 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005936 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005937 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5938 // Not dropped frame, as initial frame drop is disabled by now.
5939 WaitForEncodedFrame(timestamp);
5940 timestamp += 9000;
5941 AdvanceTime(TimeDelta::Millis(100));
5942
5943 // Quality adaptation down.
5944 video_stream_encoder_->TriggerQualityLow();
5945
5946 // Adaptation has an effect.
5947 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
5948 5000);
5949
5950 // Frame isn't dropped as initial frame dropper is disabled.
5951 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5952 WaitForEncodedFrame(timestamp);
5953 timestamp += 9000;
5954 AdvanceTime(TimeDelta::Millis(100));
5955
5956 // Quality adaptation up.
5957 video_stream_encoder_->TriggerQualityHigh();
5958
5959 // Adaptation has an effect.
5960 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
5961 5000);
5962
5963 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5964 // Frame should not be dropped, as initial framedropper is off.
5965 WaitForEncodedFrame(timestamp);
5966
5967 video_stream_encoder_->Stop();
5968}
5969
Åsa Persson7f354f82021-02-04 15:52:15 +01005970TEST_F(VideoStreamEncoderTest,
5971 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
5972 const int kMinStartBps360p = 222000;
5973 fake_encoder_.SetResolutionBitrateLimits(
5974 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
5975 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
5976 800000)});
5977
5978 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5979 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
5980 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
5981 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
5982 0, 0, 0);
5983 // Frame should not be dropped, bitrate not too low for frame.
5984 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
5985 WaitForEncodedFrame(1);
5986
5987 // Incoming resolution increases, initial frame drop activates.
5988 // Frame should be dropped, link allocation too low for frame.
5989 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
5990 ExpectDroppedFrame();
5991
5992 // Expect sink_wants to specify a scaled frame.
5993 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
5994 5000);
5995 video_stream_encoder_->Stop();
5996}
5997
5998TEST_F(VideoStreamEncoderTest,
5999 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6000 const int kMinStartBps360p = 222000;
6001 fake_encoder_.SetResolutionBitrateLimits(
6002 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6003 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6004 800000)});
6005
6006 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6007 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6008 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6009 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6010 0, 0, 0);
6011 // Frame should not be dropped, bitrate not too low for frame.
6012 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6013 WaitForEncodedFrame(1);
6014
6015 // Incoming resolution increases, initial frame drop activates.
6016 // Frame should be dropped, link allocation not too low for frame.
6017 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6018 WaitForEncodedFrame(2);
6019
6020 video_stream_encoder_->Stop();
6021}
6022
Åsa Perssone644a032019-11-08 15:56:00 +01006023TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
6024 webrtc::test::ScopedFieldTrials field_trials(
Åsa Persson06defc42021-09-10 15:28:48 +02006025 "WebRTC-Video-QualityRampupSettings/"
6026 "min_pixels:921600,min_duration_ms:2000/");
6027
6028 const int kWidth = 1280;
6029 const int kHeight = 720;
6030 const int kFps = 10;
6031 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006032
6033 // Reset encoder for field trials to take effect.
6034 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 10:07:11 +02006035 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 11:34:32 +02006036 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01006037 ConfigureEncoder(std::move(config));
6038 fake_encoder_.SetQp(kQpLow);
6039
6040 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006041 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01006042 source.set_adaptation_enabled(true);
6043 video_stream_encoder_->SetSource(&source,
6044 DegradationPreference::MAINTAIN_FRAMERATE);
6045
6046 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 10:07:11 +02006047 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 18:49:07 +02006048 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006049 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006050
6051 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 15:28:48 +02006052 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006053 int64_t timestamp_ms = kFrameIntervalMs;
6054 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6055 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02006056 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6057 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01006058
6059 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02006060 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6061 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006062
Artem Titovab30d722021-07-27 16:22:11 +02006063 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 15:28:48 +02006064 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 15:56:00 +01006065 for (size_t i = 1; i <= 10; i++) {
6066 timestamp_ms += kFrameIntervalMs;
6067 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6068 WaitForEncodedFrame(timestamp_ms);
6069 }
Åsa Persson06defc42021-09-10 15:28:48 +02006070
6071 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6072 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6073 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6074 timestamp_ms += kFrameIntervalMs;
6075 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6076 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 15:56:00 +01006077 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6078 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6079
Åsa Persson06defc42021-09-10 15:28:48 +02006080 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 15:56:00 +01006081 timestamp_ms += kFrameIntervalMs;
6082 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6083 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006084 // The ramp-up code involves the adaptation queue, give it time to execute.
6085 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006086 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006087 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006088
6089 // Frame should not be adapted.
6090 timestamp_ms += kFrameIntervalMs;
6091 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6092 WaitForEncodedFrame(kWidth, kHeight);
6093 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6094
6095 video_stream_encoder_->Stop();
6096}
6097
mflodmancc3d4422017-08-03 08:27:51 -07006098TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006099 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Ilya Nikolaevskiy483b31c2021-02-03 17:19:31 +01006100 webrtc::test::ScopedFieldTrials field_trials(
6101 "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006102 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006103 source.set_adaptation_enabled(true);
6104 video_stream_encoder_->SetSource(&source,
6105 DegradationPreference::MAINTAIN_FRAMERATE);
6106 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006107 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006108 fake_encoder_.SetQp(kQpHigh + 1);
6109 const int kWidth = 1280;
6110 const int kHeight = 720;
6111 const int64_t kFrameIntervalMs = 100;
6112 int64_t timestamp_ms = kFrameIntervalMs;
6113 for (size_t i = 1; i <= 100; i++) {
6114 timestamp_ms += kFrameIntervalMs;
6115 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6116 WaitForEncodedFrame(timestamp_ms);
6117 }
6118 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6119 // for the first time.
6120 // TODO(eshr): We should avoid these waits by using threads with simulated
6121 // time.
6122 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6123 2000 * 2.5 * 2);
6124 timestamp_ms += kFrameIntervalMs;
6125 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6126 WaitForEncodedFrame(timestamp_ms);
6127 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6128 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6129 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6130
6131 // Disable Quality scaling by turning off scaler on the encoder and
6132 // reconfiguring.
6133 fake_encoder_.SetQualityScaling(false);
6134 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6135 kMaxPayloadLength);
6136 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Markus Handell28c71802021-11-08 10:11:55 +01006137 AdvanceTime(TimeDelta::Zero());
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006138 // Since we turned off the quality scaler, the adaptations made by it are
6139 // removed.
6140 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6141 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6142
6143 video_stream_encoder_->Stop();
6144}
6145
6146TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006147 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6148 const int kTooSmallWidth = 10;
6149 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006150 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006151 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006152
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006153 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006154 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006155 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006156 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006157 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006158 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6159
6160 // Trigger adapt down, too small frame, expect no change.
6161 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006162 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006163 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006164 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006165 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6166 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6167
mflodmancc3d4422017-08-03 08:27:51 -07006168 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006169}
6170
mflodmancc3d4422017-08-03 08:27:51 -07006171TEST_F(VideoStreamEncoderTest,
6172 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006173 const int kTooSmallWidth = 10;
6174 const int kTooSmallHeight = 10;
6175 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006176 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006177 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006178
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006179 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006180 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006181 video_stream_encoder_->SetSource(&source,
6182 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006183 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006184 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6185 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6186
6187 // Trigger adapt down, expect limited framerate.
6188 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006189 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006190 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006191 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006192 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6193 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6194 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6195
6196 // Trigger adapt down, too small frame, expect no change.
6197 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006198 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006199 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006200 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006201 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6202 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6203 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6204
mflodmancc3d4422017-08-03 08:27:51 -07006205 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006206}
6207
mflodmancc3d4422017-08-03 08:27:51 -07006208TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006209 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006210 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006211 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006212 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006213 const int kFrameWidth = 1280;
6214 const int kFrameHeight = 720;
6215 video_source_.IncomingCapturedFrame(
6216 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006217 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006218 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006219}
6220
sprangb1ca0732017-02-01 08:38:12 -08006221// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006222TEST_F(VideoStreamEncoderTest,
6223 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006224 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006225 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006226
6227 const int kFrameWidth = 1280;
6228 const int kFrameHeight = 720;
6229 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006230 // requested by
6231 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006232 video_source_.set_adaptation_enabled(true);
6233
6234 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006235 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006236 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006237
6238 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006239 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006240 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006241 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006242 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006243
asaperssonfab67072017-04-04 05:51:49 -07006244 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006245 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006246 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006247 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006248 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006249
mflodmancc3d4422017-08-03 08:27:51 -07006250 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006251}
sprangfe627f32017-03-29 08:24:59 -07006252
mflodmancc3d4422017-08-03 08:27:51 -07006253TEST_F(VideoStreamEncoderTest,
6254 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006255 const int kFrameWidth = 1280;
6256 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006257
Henrik Boström381d1092020-05-12 18:49:07 +02006258 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006259 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006260 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006261 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006262 video_source_.set_adaptation_enabled(true);
6263
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006264 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006265
6266 video_source_.IncomingCapturedFrame(
6267 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006268 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006269
6270 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006271 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006272
6273 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006274 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006275 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006276 video_source_.IncomingCapturedFrame(
6277 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006278 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006279 }
6280
6281 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006282 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006283 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006284 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006285 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006286 video_source_.IncomingCapturedFrame(
6287 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006288 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006289 ++num_frames_dropped;
6290 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006291 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006292 }
6293 }
6294
sprang4847ae62017-06-27 07:06:52 -07006295 // Add some slack to account for frames dropped by the frame dropper.
6296 const int kErrorMargin = 1;
6297 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006298 kErrorMargin);
6299
6300 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006301 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006302 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006303 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006304 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006305 video_source_.IncomingCapturedFrame(
6306 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006307 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006308 ++num_frames_dropped;
6309 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006310 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006311 }
6312 }
sprang4847ae62017-06-27 07:06:52 -07006313 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006314 kErrorMargin);
6315
6316 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006317 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006318 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006319 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006320 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006321 video_source_.IncomingCapturedFrame(
6322 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006323 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006324 ++num_frames_dropped;
6325 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006326 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006327 }
6328 }
sprang4847ae62017-06-27 07:06:52 -07006329 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006330 kErrorMargin);
6331
6332 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006333 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006334 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006335 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006336 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006337 video_source_.IncomingCapturedFrame(
6338 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006339 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006340 ++num_frames_dropped;
6341 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006342 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006343 }
6344 }
6345 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6346
mflodmancc3d4422017-08-03 08:27:51 -07006347 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006348}
6349
mflodmancc3d4422017-08-03 08:27:51 -07006350TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006351 const int kFramerateFps = 5;
6352 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006353 const int kFrameWidth = 1280;
6354 const int kFrameHeight = 720;
6355
sprang4847ae62017-06-27 07:06:52 -07006356 // Reconfigure encoder with two temporal layers and screensharing, which will
6357 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006358 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006359
Henrik Boström381d1092020-05-12 18:49:07 +02006360 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006361 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006362 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006363 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006364 video_source_.set_adaptation_enabled(true);
6365
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006366 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006367
6368 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006369 rtc::VideoSinkWants last_wants;
6370 do {
6371 last_wants = video_source_.sink_wants();
6372
sprangc5d62e22017-04-02 23:53:04 -07006373 // Insert frames to get a new fps estimate...
6374 for (int j = 0; j < kFramerateFps; ++j) {
6375 video_source_.IncomingCapturedFrame(
6376 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006377 if (video_source_.last_sent_width()) {
6378 sink_.WaitForEncodedFrame(timestamp_ms);
6379 }
sprangc5d62e22017-04-02 23:53:04 -07006380 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006381 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006382 }
6383 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006384 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006385 } while (video_source_.sink_wants().max_framerate_fps <
6386 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006387
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006388 EXPECT_THAT(video_source_.sink_wants(),
6389 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006390
mflodmancc3d4422017-08-03 08:27:51 -07006391 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006392}
asaperssonf7e294d2017-06-13 23:25:22 -07006393
mflodmancc3d4422017-08-03 08:27:51 -07006394TEST_F(VideoStreamEncoderTest,
6395 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006396 const int kWidth = 1280;
6397 const int kHeight = 720;
6398 const int64_t kFrameIntervalMs = 150;
6399 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006400 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006401 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006402
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006403 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006404 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006405 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006406 video_stream_encoder_->SetSource(&source,
6407 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006408 timestamp_ms += kFrameIntervalMs;
6409 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006410 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006411 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006412 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6413 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6414 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6415
6416 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006417 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006418 timestamp_ms += kFrameIntervalMs;
6419 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006420 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006421 EXPECT_THAT(source.sink_wants(),
6422 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006423 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6424 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6425 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6426
6427 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006428 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006429 timestamp_ms += kFrameIntervalMs;
6430 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006431 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006432 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006433 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6434 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6435 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6436
6437 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006438 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006439 timestamp_ms += kFrameIntervalMs;
6440 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006441 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006442 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006443 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6444 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6445 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6446
6447 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006448 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006449 timestamp_ms += kFrameIntervalMs;
6450 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006451 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006452 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006453 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6454 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6455 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6456
6457 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006458 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006459 timestamp_ms += kFrameIntervalMs;
6460 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006461 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006462 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006463 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6464 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6465 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6466
6467 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006468 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006469 timestamp_ms += kFrameIntervalMs;
6470 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006471 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006472 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006473 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6474 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6475 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6476
6477 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006478 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006479 timestamp_ms += kFrameIntervalMs;
6480 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006481 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006482 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006483 rtc::VideoSinkWants last_wants = source.sink_wants();
6484 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6485 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6486 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6487
6488 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006489 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006490 timestamp_ms += kFrameIntervalMs;
6491 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006492 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006493 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006494 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6495 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6496 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6497
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006498 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006499 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006500 timestamp_ms += kFrameIntervalMs;
6501 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006502 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006503 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006504 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6505 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6506 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6507
6508 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006509 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006510 timestamp_ms += kFrameIntervalMs;
6511 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006512 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006513 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006514 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6515 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6516 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6517
6518 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006519 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006520 timestamp_ms += kFrameIntervalMs;
6521 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006522 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006523 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006524 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6525 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6526 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6527
6528 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006529 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006530 timestamp_ms += kFrameIntervalMs;
6531 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006532 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006533 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006534 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6535 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6536 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6537
6538 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006539 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006540 timestamp_ms += kFrameIntervalMs;
6541 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006542 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006543 EXPECT_THAT(source.sink_wants(), FpsMax());
6544 EXPECT_EQ(source.sink_wants().max_pixel_count,
6545 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006546 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6547 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6548 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6549
6550 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006551 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006552 timestamp_ms += kFrameIntervalMs;
6553 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006554 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006555 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006556 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6557 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6558 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6559
Åsa Persson30ab0152019-08-27 12:22:33 +02006560 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006561 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006562 timestamp_ms += kFrameIntervalMs;
6563 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006564 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006565 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006566 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006567 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6568 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6569 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6570
6571 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006572 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006573 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006574 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6575
mflodmancc3d4422017-08-03 08:27:51 -07006576 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006577}
6578
mflodmancc3d4422017-08-03 08:27:51 -07006579TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006580 const int kWidth = 1280;
6581 const int kHeight = 720;
6582 const int64_t kFrameIntervalMs = 150;
6583 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006584 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006585 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006586
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006587 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006588 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006589 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006590 video_stream_encoder_->SetSource(&source,
6591 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006592 timestamp_ms += kFrameIntervalMs;
6593 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006594 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006595 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006596 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6597 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6598 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6599 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6600 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6601 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6602
6603 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006604 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006605 timestamp_ms += kFrameIntervalMs;
6606 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006607 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006608 EXPECT_THAT(source.sink_wants(),
6609 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006610 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6611 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6612 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6613 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6614 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6615 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6616
6617 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006618 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006619 timestamp_ms += kFrameIntervalMs;
6620 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006621 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006622 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006623 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6624 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6625 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6626 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6627 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6628 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6629
6630 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006631 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006632 timestamp_ms += kFrameIntervalMs;
6633 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006634 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006635 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006636 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006637 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6638 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6639 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6640 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6641 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6642
Evan Shrubsole64469032020-06-11 10:45:29 +02006643 // Trigger cpu adapt up, expect no change since QP is most limited.
6644 {
6645 // Store current sink wants since we expect no change and if there is no
6646 // change then last_wants() is not updated.
6647 auto previous_sink_wants = source.sink_wants();
6648 video_stream_encoder_->TriggerCpuUnderuse();
6649 timestamp_ms += kFrameIntervalMs;
6650 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6651 WaitForEncodedFrame(timestamp_ms);
6652 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6653 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6654 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6655 }
6656
6657 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6658 video_stream_encoder_->TriggerQualityHigh();
6659 timestamp_ms += kFrameIntervalMs;
6660 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6661 WaitForEncodedFrame(timestamp_ms);
6662 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6663 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6664 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6665 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6666 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6667 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6668 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6669
6670 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6671 // expect increased resolution (960x540@30fps).
6672 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006673 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006674 timestamp_ms += kFrameIntervalMs;
6675 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006676 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006677 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006678 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6679 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6680 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6681 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6682 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006683 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006684
Evan Shrubsole64469032020-06-11 10:45:29 +02006685 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6686 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006687 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006688 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006689 timestamp_ms += kFrameIntervalMs;
6690 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006691 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006692 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006693 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006694 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6695 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6696 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6697 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6698 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006699 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006700
6701 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006702 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006703 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006704 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006705 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006706
mflodmancc3d4422017-08-03 08:27:51 -07006707 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006708}
6709
mflodmancc3d4422017-08-03 08:27:51 -07006710TEST_F(VideoStreamEncoderTest,
6711 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006712 const int kWidth = 640;
6713 const int kHeight = 360;
6714 const int kFpsLimit = 15;
6715 const int64_t kFrameIntervalMs = 150;
6716 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006717 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006718 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006719
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006720 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006721 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006722 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006723 video_stream_encoder_->SetSource(&source,
6724 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006725 timestamp_ms += kFrameIntervalMs;
6726 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006727 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006728 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006729 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6730 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6731 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6732 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6733 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6734 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6735
6736 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006737 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006738 timestamp_ms += kFrameIntervalMs;
6739 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006740 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006741 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006742 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6743 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6744 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6745 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6746 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6747 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6748
6749 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006750 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006751 timestamp_ms += kFrameIntervalMs;
6752 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006753 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006754 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006755 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006756 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006757 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6758 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6759 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6760 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6761
Evan Shrubsole64469032020-06-11 10:45:29 +02006762 // Trigger cpu adapt up, expect no change because quality is most limited.
6763 {
6764 auto previous_sink_wants = source.sink_wants();
6765 // Store current sink wants since we expect no change ind if there is no
6766 // change then last__wants() is not updated.
6767 video_stream_encoder_->TriggerCpuUnderuse();
6768 timestamp_ms += kFrameIntervalMs;
6769 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6770 WaitForEncodedFrame(timestamp_ms);
6771 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6772 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6773 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6774 }
6775
6776 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6777 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006778 timestamp_ms += kFrameIntervalMs;
6779 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006780 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006781 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006782 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6783 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6784 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006785 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6786 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6787 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006788
Evan Shrubsole64469032020-06-11 10:45:29 +02006789 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006790 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006791 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006792 timestamp_ms += kFrameIntervalMs;
6793 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006794 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006795 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006796 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6797 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6798 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6799 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6800 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006801 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006802
6803 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006804 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006805 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006806 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006807 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006808
mflodmancc3d4422017-08-03 08:27:51 -07006809 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006810}
6811
mflodmancc3d4422017-08-03 08:27:51 -07006812TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006813 const int kFrameWidth = 1920;
6814 const int kFrameHeight = 1080;
6815 // 3/4 of 1920.
6816 const int kAdaptedFrameWidth = 1440;
6817 // 3/4 of 1080 rounded down to multiple of 4.
6818 const int kAdaptedFrameHeight = 808;
6819 const int kFramerate = 24;
6820
Henrik Boström381d1092020-05-12 18:49:07 +02006821 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006822 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006823 // Trigger reconfigure encoder (without resetting the entire instance).
6824 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006825 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6826 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02006827 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 06:53:48 -07006828 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006829 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006830 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006831 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006832 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006833
6834 video_source_.set_adaptation_enabled(true);
6835
6836 video_source_.IncomingCapturedFrame(
6837 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006838 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006839
6840 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006841 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006842 video_source_.IncomingCapturedFrame(
6843 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006844 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006845
mflodmancc3d4422017-08-03 08:27:51 -07006846 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006847}
6848
mflodmancc3d4422017-08-03 08:27:51 -07006849TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006850 const int kFrameWidth = 1280;
6851 const int kFrameHeight = 720;
6852 const int kLowFps = 2;
6853 const int kHighFps = 30;
6854
Henrik Boström381d1092020-05-12 18:49:07 +02006855 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006856 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006857
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006858 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006859 max_framerate_ = kLowFps;
6860
6861 // Insert 2 seconds of 2fps video.
6862 for (int i = 0; i < kLowFps * 2; ++i) {
6863 video_source_.IncomingCapturedFrame(
6864 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6865 WaitForEncodedFrame(timestamp_ms);
6866 timestamp_ms += 1000 / kLowFps;
6867 }
6868
6869 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006870 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006871 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006872 video_source_.IncomingCapturedFrame(
6873 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6874 WaitForEncodedFrame(timestamp_ms);
6875 timestamp_ms += 1000 / kLowFps;
6876
6877 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6878
6879 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006880 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006881 const int kFrameIntervalMs = 1000 / kHighFps;
6882 max_framerate_ = kHighFps;
6883 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6884 video_source_.IncomingCapturedFrame(
6885 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6886 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6887 // be dropped if the encoder hans't been updated with the new higher target
6888 // framerate yet, causing it to overshoot the target bitrate and then
6889 // suffering the wrath of the media optimizer.
6890 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6891 timestamp_ms += kFrameIntervalMs;
6892 }
6893
6894 // Don expect correct measurement just yet, but it should be higher than
6895 // before.
6896 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6897
mflodmancc3d4422017-08-03 08:27:51 -07006898 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006899}
6900
mflodmancc3d4422017-08-03 08:27:51 -07006901TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07006902 const int kFrameWidth = 1280;
6903 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 15:09:05 +02006904 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01006905 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02006906 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07006907
Henrik Boström381d1092020-05-12 18:49:07 +02006908 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006909 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006910 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07006911
6912 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006913 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006914 video_source_.IncomingCapturedFrame(
6915 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6916 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02006917 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006918
6919 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02006920 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006921 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07006922
6923 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02006924 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006925 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07006926
Per Kjellanderdcef6412020-10-07 15:09:05 +02006927 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07006928 video_source_.IncomingCapturedFrame(
6929 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6930 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02006931 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006932
mflodmancc3d4422017-08-03 08:27:51 -07006933 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006934}
ilnik6b826ef2017-06-16 06:53:48 -07006935
Niels Möller4db138e2018-04-19 09:04:13 +02006936TEST_F(VideoStreamEncoderTest,
6937 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
6938 const int kFrameWidth = 1280;
6939 const int kFrameHeight = 720;
6940 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02006941 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006942 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006943 video_source_.IncomingCapturedFrame(
6944 CreateFrame(1, kFrameWidth, kFrameHeight));
6945 WaitForEncodedFrame(1);
6946 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6947 .low_encode_usage_threshold_percent,
6948 default_options.low_encode_usage_threshold_percent);
6949 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6950 .high_encode_usage_threshold_percent,
6951 default_options.high_encode_usage_threshold_percent);
6952 video_stream_encoder_->Stop();
6953}
6954
6955TEST_F(VideoStreamEncoderTest,
6956 HigherCpuAdaptationThresholdsForHardwareEncoder) {
6957 const int kFrameWidth = 1280;
6958 const int kFrameHeight = 720;
6959 CpuOveruseOptions hardware_options;
6960 hardware_options.low_encode_usage_threshold_percent = 150;
6961 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01006962 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02006963
Henrik Boström381d1092020-05-12 18:49:07 +02006964 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006965 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02006966 video_source_.IncomingCapturedFrame(
6967 CreateFrame(1, kFrameWidth, kFrameHeight));
6968 WaitForEncodedFrame(1);
6969 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6970 .low_encode_usage_threshold_percent,
6971 hardware_options.low_encode_usage_threshold_percent);
6972 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6973 .high_encode_usage_threshold_percent,
6974 hardware_options.high_encode_usage_threshold_percent);
6975 video_stream_encoder_->Stop();
6976}
6977
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01006978TEST_F(VideoStreamEncoderTest,
6979 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
6980 const int kFrameWidth = 1280;
6981 const int kFrameHeight = 720;
6982
6983 const CpuOveruseOptions default_options;
6984 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006985 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01006986 video_source_.IncomingCapturedFrame(
6987 CreateFrame(1, kFrameWidth, kFrameHeight));
6988 WaitForEncodedFrame(1);
6989 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6990 .low_encode_usage_threshold_percent,
6991 default_options.low_encode_usage_threshold_percent);
6992 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
6993 .high_encode_usage_threshold_percent,
6994 default_options.high_encode_usage_threshold_percent);
6995
6996 CpuOveruseOptions hardware_options;
6997 hardware_options.low_encode_usage_threshold_percent = 150;
6998 hardware_options.high_encode_usage_threshold_percent = 200;
6999 fake_encoder_.SetIsHardwareAccelerated(true);
7000
7001 video_source_.IncomingCapturedFrame(
7002 CreateFrame(2, kFrameWidth, kFrameHeight));
7003 WaitForEncodedFrame(2);
7004
7005 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7006 .low_encode_usage_threshold_percent,
7007 hardware_options.low_encode_usage_threshold_percent);
7008 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7009 .high_encode_usage_threshold_percent,
7010 hardware_options.high_encode_usage_threshold_percent);
7011
7012 video_stream_encoder_->Stop();
7013}
7014
Niels Möller6bb5ab92019-01-11 11:11:10 +01007015TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7016 const int kFrameWidth = 320;
7017 const int kFrameHeight = 240;
7018 const int kFps = 30;
Asa Persson606d3cb2021-10-04 10:07:11 +02007019 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007020 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7021
Henrik Boström381d1092020-05-12 18:49:07 +02007022 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007023 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007024
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007025 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007026 max_framerate_ = kFps;
7027
7028 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7029 fake_encoder_.SimulateOvershoot(1.0);
7030 int num_dropped = 0;
7031 for (int i = 0; i < kNumFramesInRun; ++i) {
7032 video_source_.IncomingCapturedFrame(
7033 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7034 // Wait up to two frame durations for a frame to arrive.
7035 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7036 ++num_dropped;
7037 }
7038 timestamp_ms += 1000 / kFps;
7039 }
7040
Erik Språnga8d48ab2019-02-08 14:17:40 +01007041 // Framerate should be measured to be near the expected target rate.
7042 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7043
7044 // Frame drops should be within 5% of expected 0%.
7045 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007046
7047 // Make encoder produce frames at double the expected bitrate during 3 seconds
7048 // of video, verify number of drops. Rate needs to be slightly changed in
7049 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01007050 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02007051 const RateControlSettings trials =
7052 RateControlSettings::ParseFromFieldTrials();
7053 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01007054 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02007055 // frame dropping since the adjuter will try to just lower the target
7056 // bitrate rather than drop frames. If network headroom can be used, it
7057 // doesn't push back as hard so we don't need quite as much overshoot.
7058 // These numbers are unfortunately a bit magical but there's not trivial
7059 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01007060 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01007061 }
7062 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02007063 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007064 kTargetBitrate + DataRate::KilobitsPerSec(1),
7065 kTargetBitrate + DataRate::KilobitsPerSec(1),
7066 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007067 num_dropped = 0;
7068 for (int i = 0; i < kNumFramesInRun; ++i) {
7069 video_source_.IncomingCapturedFrame(
7070 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7071 // Wait up to two frame durations for a frame to arrive.
7072 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7073 ++num_dropped;
7074 }
7075 timestamp_ms += 1000 / kFps;
7076 }
7077
Henrik Boström381d1092020-05-12 18:49:07 +02007078 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007079 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007080
7081 // Target framerate should be still be near the expected target, despite
7082 // the frame drops.
7083 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7084
7085 // Frame drops should be within 5% of expected 50%.
7086 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007087
7088 video_stream_encoder_->Stop();
7089}
7090
7091TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7092 const int kFrameWidth = 320;
7093 const int kFrameHeight = 240;
7094 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 10:07:11 +02007095 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007096
7097 ASSERT_GT(max_framerate_, kActualInputFps);
7098
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007099 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007100 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007101 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007102 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007103
7104 // Insert 3 seconds of video, with an input fps lower than configured max.
7105 for (int i = 0; i < kActualInputFps * 3; ++i) {
7106 video_source_.IncomingCapturedFrame(
7107 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7108 // Wait up to two frame durations for a frame to arrive.
7109 WaitForEncodedFrame(timestamp_ms);
7110 timestamp_ms += 1000 / kActualInputFps;
7111 }
7112
7113 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7114
7115 video_stream_encoder_->Stop();
7116}
7117
Markus Handell9a478b52021-11-18 16:07:01 +01007118TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007119 VideoFrame::UpdateRect rect;
Markus Handell9a478b52021-11-18 16:07:01 +01007120 test::FrameForwarder source;
7121 video_stream_encoder_->SetSource(&source,
7122 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02007123 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007124 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007125
Markus Handell9a478b52021-11-18 16:07:01 +01007126 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(1, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007127 WaitForEncodedFrame(1);
7128 // On the very first frame full update should be forced.
7129 rect = fake_encoder_.GetLastUpdateRect();
7130 EXPECT_EQ(rect.offset_x, 0);
7131 EXPECT_EQ(rect.offset_y, 0);
7132 EXPECT_EQ(rect.height, codec_height_);
7133 EXPECT_EQ(rect.width, codec_width_);
Markus Handell9a478b52021-11-18 16:07:01 +01007134 // Frame with NTP timestamp 2 will be dropped due to outstanding frames
7135 // scheduled for processing during encoder queue processing of frame 2.
7136 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(2, nullptr, 1));
7137 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(3, nullptr, 10));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007138 WaitForEncodedFrame(3);
7139 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7140 rect = fake_encoder_.GetLastUpdateRect();
7141 EXPECT_EQ(rect.offset_x, 1);
7142 EXPECT_EQ(rect.offset_y, 0);
7143 EXPECT_EQ(rect.width, 10);
7144 EXPECT_EQ(rect.height, 1);
7145
Markus Handell9a478b52021-11-18 16:07:01 +01007146 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(4, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007147 WaitForEncodedFrame(4);
7148 // Previous frame was encoded, so no accumulation should happen.
7149 rect = fake_encoder_.GetLastUpdateRect();
7150 EXPECT_EQ(rect.offset_x, 0);
7151 EXPECT_EQ(rect.offset_y, 0);
7152 EXPECT_EQ(rect.width, 1);
7153 EXPECT_EQ(rect.height, 1);
7154
7155 video_stream_encoder_->Stop();
7156}
7157
Erik Språngd7329ca2019-02-21 21:19:53 +01007158TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007159 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007160 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007161
7162 // First frame is always keyframe.
7163 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7164 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007165 EXPECT_THAT(
7166 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007167 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007168
7169 // Insert delta frame.
7170 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7171 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007172 EXPECT_THAT(
7173 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007174 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007175
7176 // Request next frame be a key-frame.
7177 video_stream_encoder_->SendKeyFrame();
7178 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7179 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007180 EXPECT_THAT(
7181 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007182 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007183
7184 video_stream_encoder_->Stop();
7185}
7186
7187TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7188 // Setup simulcast with three streams.
7189 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007190 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007191 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7192 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007193 // Wait for all three layers before triggering event.
7194 sink_.SetNumExpectedLayers(3);
7195
7196 // First frame is always keyframe.
7197 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7198 WaitForEncodedFrame(1);
7199 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007200 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7201 VideoFrameType::kVideoFrameKey,
7202 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007203
7204 // Insert delta frame.
7205 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7206 WaitForEncodedFrame(2);
7207 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007208 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7209 VideoFrameType::kVideoFrameDelta,
7210 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007211
7212 // Request next frame be a key-frame.
7213 // Only first stream is configured to produce key-frame.
7214 video_stream_encoder_->SendKeyFrame();
7215 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7216 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007217
7218 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7219 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007220 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007221 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007222 VideoFrameType::kVideoFrameKey,
7223 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007224
7225 video_stream_encoder_->Stop();
7226}
7227
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007228TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007229 // SPS contains VUI with restrictions on the maximum number of reordered
7230 // pictures, there is no need to rewrite the bitstream to enable faster
7231 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007232 ResetEncoder("H264", 1, 1, 1, false);
7233
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007234 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007235 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007236 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007237
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007238 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 10:07:11 +02007239 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007240
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007241 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7242 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007243
7244 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007245 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007246
7247 video_stream_encoder_->Stop();
7248}
7249
7250TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007251 // SPS does not contain VUI, the bitstream is will be rewritten with added
7252 // VUI with restrictions on the maximum number of reordered pictures to
7253 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007254 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7255 0x00, 0x00, 0x03, 0x03, 0xF4,
7256 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007257 ResetEncoder("H264", 1, 1, 1, false);
7258
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007259 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007260 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007261 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007262
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007263 fake_encoder_.SetEncodedImageData(
7264 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007265
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007266 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7267 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007268
7269 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007270 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007271
7272 video_stream_encoder_->Stop();
7273}
7274
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007275TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7276 const int kFrameWidth = 1280;
7277 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 10:07:11 +02007278 const DataRate kTargetBitrate =
7279 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007280
Henrik Boström381d1092020-05-12 18:49:07 +02007281 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007282 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007283 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7284
7285 // Insert a first video frame. It should be dropped because of downscale in
7286 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007287 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007288 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7289 frame.set_rotation(kVideoRotation_270);
7290 video_source_.IncomingCapturedFrame(frame);
7291
7292 ExpectDroppedFrame();
7293
7294 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007295 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007296 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7297 frame.set_rotation(kVideoRotation_90);
7298 video_source_.IncomingCapturedFrame(frame);
7299
7300 WaitForEncodedFrame(timestamp_ms);
7301 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7302
7303 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007304 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007305 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7306 frame.set_rotation(kVideoRotation_180);
7307 video_source_.IncomingCapturedFrame(frame);
7308
7309 WaitForEncodedFrame(timestamp_ms);
7310 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7311
7312 video_stream_encoder_->Stop();
7313}
7314
Erik Språng5056af02019-09-02 15:53:11 +02007315TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7316 const int kFrameWidth = 320;
7317 const int kFrameHeight = 180;
7318
7319 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007320 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007321 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7322 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7323 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007324 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007325 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007326 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007327
7328 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007329 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007330 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7331 frame.set_rotation(kVideoRotation_270);
7332 video_source_.IncomingCapturedFrame(frame);
7333 WaitForEncodedFrame(timestamp_ms);
7334
7335 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 10:07:11 +02007336 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007337 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7338 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007339 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007340 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007341 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007342 /*link_allocation=*/target_rate,
7343 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007344 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007345 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007346 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7347
7348 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7349 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7350 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007351 DataRate allocation_sum =
7352 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007353 EXPECT_EQ(min_rate, allocation_sum);
7354 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7355
7356 video_stream_encoder_->Stop();
7357}
7358
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007359TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007360 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007361 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007362 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007363 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007364 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7365 WaitForEncodedFrame(1);
7366
7367 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7368 ASSERT_TRUE(prev_rate_settings.has_value());
7369 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7370 kDefaultFramerate);
7371
7372 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7373 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7374 timestamp_ms += 1000 / kDefaultFramerate;
7375 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7376 WaitForEncodedFrame(timestamp_ms);
7377 }
7378 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7379 kDefaultFramerate);
7380 // Capture larger frame to trigger a reconfigure.
7381 codec_height_ *= 2;
7382 codec_width_ *= 2;
7383 timestamp_ms += 1000 / kDefaultFramerate;
7384 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7385 WaitForEncodedFrame(timestamp_ms);
7386
7387 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7388 auto current_rate_settings =
7389 fake_encoder_.GetAndResetLastRateControlSettings();
7390 // Ensure we have actually reconfigured twice
7391 // The rate settings should have been set again even though
7392 // they haven't changed.
7393 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007394 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007395
7396 video_stream_encoder_->Stop();
7397}
7398
philipeld9cc8c02019-09-16 14:53:40 +02007399struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007400 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
7401 MOCK_METHOD(void, RequestEncoderSwitch, (const Config& conf), (override));
7402 MOCK_METHOD(void,
7403 RequestEncoderSwitch,
7404 (const webrtc::SdpVideoFormat& format),
7405 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007406};
7407
philipel9b058032020-02-10 11:30:00 +01007408TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7409 constexpr int kDontCare = 100;
7410 StrictMock<MockEncoderSelector> encoder_selector;
7411 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7412 &fake_encoder_, &encoder_selector);
7413 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7414
7415 // Reset encoder for new configuration to take effect.
7416 ConfigureEncoder(video_encoder_config_.Copy());
7417
7418 EXPECT_CALL(encoder_selector, OnCurrentEncoder(_));
7419
7420 video_source_.IncomingCapturedFrame(
7421 CreateFrame(kDontCare, kDontCare, kDontCare));
Markus Handell28c71802021-11-08 10:11:55 +01007422 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007423 video_stream_encoder_->Stop();
7424
7425 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7426 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007427 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7428 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007429 video_stream_encoder_.reset();
7430}
7431
7432TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7433 constexpr int kDontCare = 100;
7434
7435 NiceMock<MockEncoderSelector> encoder_selector;
7436 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7437 video_send_config_.encoder_settings.encoder_switch_request_callback =
7438 &switch_callback;
7439 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7440 &fake_encoder_, &encoder_selector);
7441 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7442
7443 // Reset encoder for new configuration to take effect.
7444 ConfigureEncoder(video_encoder_config_.Copy());
7445
Mirta Dvornicic4f34d782020-02-26 13:01:19 +01007446 ON_CALL(encoder_selector, OnAvailableBitrate(_))
philipel9b058032020-02-10 11:30:00 +01007447 .WillByDefault(Return(SdpVideoFormat("AV1")));
7448 EXPECT_CALL(switch_callback,
7449 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(
7450 Field(&SdpVideoFormat::name, "AV1"))));
7451
Henrik Boström381d1092020-05-12 18:49:07 +02007452 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007453 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7454 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7455 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007456 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007457 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007458 /*cwnd_reduce_ratio=*/0);
Markus Handell28c71802021-11-08 10:11:55 +01007459 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007460
7461 video_stream_encoder_->Stop();
7462}
7463
7464TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7465 constexpr int kSufficientBitrateToNotDrop = 1000;
7466 constexpr int kDontCare = 100;
7467
7468 NiceMock<MockVideoEncoder> video_encoder;
7469 NiceMock<MockEncoderSelector> encoder_selector;
7470 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7471 video_send_config_.encoder_settings.encoder_switch_request_callback =
7472 &switch_callback;
7473 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7474 &video_encoder, &encoder_selector);
7475 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7476
7477 // Reset encoder for new configuration to take effect.
7478 ConfigureEncoder(video_encoder_config_.Copy());
7479
7480 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7481 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7482 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007483 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007484 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7485 /*stable_target_bitrate=*/
7486 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7487 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007488 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007489 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007490 /*cwnd_reduce_ratio=*/0);
7491
7492 ON_CALL(video_encoder, Encode(_, _))
7493 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7494 ON_CALL(encoder_selector, OnEncoderBroken())
7495 .WillByDefault(Return(SdpVideoFormat("AV2")));
7496
7497 rtc::Event encode_attempted;
7498 EXPECT_CALL(switch_callback,
7499 RequestEncoderSwitch(Matcher<const SdpVideoFormat&>(_)))
7500 .WillOnce([&encode_attempted](const SdpVideoFormat& format) {
7501 EXPECT_EQ(format.name, "AV2");
7502 encode_attempted.Set();
7503 });
7504
7505 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7506 encode_attempted.Wait(3000);
7507
Markus Handell28c71802021-11-08 10:11:55 +01007508 AdvanceTime(TimeDelta::Zero());
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007509
philipel9b058032020-02-10 11:30:00 +01007510 video_stream_encoder_->Stop();
7511
7512 // The encoders produces by the VideoEncoderProxyFactory have a pointer back
7513 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007514 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7515 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007516 video_stream_encoder_.reset();
7517}
7518
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007519TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007520 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007521 const int kFrameWidth = 320;
7522 const int kFrameHeight = 180;
7523
7524 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007525 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007526 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007527 /*target_bitrate=*/rate,
7528 /*stable_target_bitrate=*/rate,
7529 /*link_allocation=*/rate,
7530 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007531 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007532 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007533
7534 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007535 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007536 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7537 frame.set_rotation(kVideoRotation_270);
7538 video_source_.IncomingCapturedFrame(frame);
7539 WaitForEncodedFrame(timestamp_ms);
7540 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7541
7542 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007543 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007544 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007545 /*target_bitrate=*/new_stable_rate,
7546 /*stable_target_bitrate=*/new_stable_rate,
7547 /*link_allocation=*/rate,
7548 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007549 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007550 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007551 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7552 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7553 video_stream_encoder_->Stop();
7554}
7555
7556TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007557 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007558 const int kFrameWidth = 320;
7559 const int kFrameHeight = 180;
7560
7561 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007562 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007563 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007564 /*target_bitrate=*/rate,
7565 /*stable_target_bitrate=*/rate,
7566 /*link_allocation=*/rate,
7567 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007568 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007569 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007570
7571 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007572 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007573 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7574 frame.set_rotation(kVideoRotation_270);
7575 video_source_.IncomingCapturedFrame(frame);
7576 WaitForEncodedFrame(timestamp_ms);
7577 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7578
7579 // Set a higher target rate without changing the link_allocation. Should not
7580 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007581 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007582 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007583 /*target_bitrate=*/rate,
7584 /*stable_target_bitrate=*/new_stable_rate,
7585 /*link_allocation=*/rate,
7586 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007587 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007588 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007589 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7590 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7591 video_stream_encoder_->Stop();
7592}
7593
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007594TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7595 test::ScopedFieldTrials field_trials(
7596 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7597 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7598 const int kFramerateFps = 30;
7599 const int kWidth = 1920;
7600 const int kHeight = 1080;
7601 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7602 // Works on screenshare mode.
7603 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7604 // We rely on the automatic resolution adaptation, but we handle framerate
7605 // adaptation manually by mocking the stats proxy.
7606 video_source_.set_adaptation_enabled(true);
7607
7608 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007609 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007610 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007611 video_stream_encoder_->SetSource(&video_source_,
7612 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007613 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007614
7615 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7616 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7617
7618 // Pass enough frames with the full update to trigger animation detection.
7619 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007620 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007621 frame.set_ntp_time_ms(timestamp_ms);
7622 frame.set_timestamp_us(timestamp_ms * 1000);
7623 video_source_.IncomingCapturedFrame(frame);
7624 WaitForEncodedFrame(timestamp_ms);
7625 }
7626
7627 // Resolution should be limited.
7628 rtc::VideoSinkWants expected;
7629 expected.max_framerate_fps = kFramerateFps;
7630 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007631 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007632
7633 // Pass one frame with no known update.
7634 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007635 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007636 frame.set_ntp_time_ms(timestamp_ms);
7637 frame.set_timestamp_us(timestamp_ms * 1000);
7638 frame.clear_update_rect();
7639
7640 video_source_.IncomingCapturedFrame(frame);
7641 WaitForEncodedFrame(timestamp_ms);
7642
7643 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007644 EXPECT_THAT(video_source_.sink_wants(),
7645 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007646
7647 video_stream_encoder_->Stop();
7648}
7649
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007650TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7651 const int kWidth = 720; // 540p adapted down.
7652 const int kHeight = 405;
7653 const int kNumFrames = 3;
7654 // Works on screenshare mode.
7655 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7656 /*num_spatial_layers=*/2, /*screenshare=*/true);
7657
7658 video_source_.set_adaptation_enabled(true);
7659
7660 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007661 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007662
7663 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7664
7665 // Pass enough frames with the full update to trigger animation detection.
7666 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007667 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007668 frame.set_ntp_time_ms(timestamp_ms);
7669 frame.set_timestamp_us(timestamp_ms * 1000);
7670 video_source_.IncomingCapturedFrame(frame);
7671 WaitForEncodedFrame(timestamp_ms);
7672 }
7673
7674 video_stream_encoder_->Stop();
7675}
7676
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007677TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7678 const float downscale_factors[] = {4.0, 2.0, 1.0};
7679 const int number_layers =
7680 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7681 VideoEncoderConfig config;
7682 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7683 for (int i = 0; i < number_layers; ++i) {
7684 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7685 config.simulcast_layers[i].active = true;
7686 }
7687 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007688 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007689 "VP8", /*max qp*/ 56, /*screencast*/ false,
7690 /*screenshare enabled*/ false);
7691 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007692 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7693 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007694
7695 // First initialization.
7696 // Encoder should be initialized. Next frame should be key frame.
7697 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7698 sink_.SetNumExpectedLayers(number_layers);
7699 int64_t timestamp_ms = kFrameIntervalMs;
7700 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7701 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007702 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007703 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7704 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7705 VideoFrameType::kVideoFrameKey,
7706 VideoFrameType::kVideoFrameKey}));
7707
7708 // Disable top layer.
7709 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7710 config.simulcast_layers[number_layers - 1].active = false;
7711 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7712 sink_.SetNumExpectedLayers(number_layers - 1);
7713 timestamp_ms += kFrameIntervalMs;
7714 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7715 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007716 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007717 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7718 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7719 VideoFrameType::kVideoFrameDelta,
7720 VideoFrameType::kVideoFrameDelta}));
7721
7722 // Re-enable top layer.
7723 // Encoder should be re-initialized. Next frame should be key frame.
7724 config.simulcast_layers[number_layers - 1].active = true;
7725 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7726 sink_.SetNumExpectedLayers(number_layers);
7727 timestamp_ms += kFrameIntervalMs;
7728 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7729 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007730 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007731 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7732 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7733 VideoFrameType::kVideoFrameKey,
7734 VideoFrameType::kVideoFrameKey}));
7735
7736 // Top layer max rate change.
7737 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7738 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7739 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7740 sink_.SetNumExpectedLayers(number_layers);
7741 timestamp_ms += kFrameIntervalMs;
7742 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7743 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007744 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007745 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7746 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7747 VideoFrameType::kVideoFrameDelta,
7748 VideoFrameType::kVideoFrameDelta}));
7749
7750 // Top layer resolution change.
7751 // Encoder should be re-initialized. Next frame should be key frame.
7752 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7753 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7754 sink_.SetNumExpectedLayers(number_layers);
7755 timestamp_ms += kFrameIntervalMs;
7756 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7757 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007758 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007759 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7760 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7761 VideoFrameType::kVideoFrameKey,
7762 VideoFrameType::kVideoFrameKey}));
7763 video_stream_encoder_->Stop();
7764}
7765
Henrik Boström1124ed12021-02-25 10:30:39 +01007766TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
7767 const int kFrameWidth = 1280;
7768 const int kFrameHeight = 720;
7769
7770 SetUp();
7771 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007772 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01007773
7774 // Capturing a frame should reconfigure the encoder and expose the encoder
7775 // resolution, which is the same as the input frame.
7776 int64_t timestamp_ms = kFrameIntervalMs;
7777 video_source_.IncomingCapturedFrame(
7778 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7779 WaitForEncodedFrame(timestamp_ms);
7780 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7781 EXPECT_THAT(video_source_.sink_wants().resolutions,
7782 ::testing::ElementsAreArray(
7783 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
7784
7785 video_stream_encoder_->Stop();
7786}
7787
7788TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
7789 // Pick downscale factors such that we never encode at full resolution - this
7790 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 16:22:11 +02007791 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 10:30:39 +01007792 // encoder should not ask for the frame resolution. This allows video frames
7793 // to have the appearence of one resolution but optimize its internal buffers
7794 // for what is actually encoded.
7795 const size_t kNumSimulcastLayers = 3u;
7796 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
7797 const int kFrameWidth = 1280;
7798 const int kFrameHeight = 720;
7799 const rtc::VideoSinkWants::FrameSize kLayer0Size(
7800 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
7801 const rtc::VideoSinkWants::FrameSize kLayer1Size(
7802 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
7803 const rtc::VideoSinkWants::FrameSize kLayer2Size(
7804 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
7805
7806 VideoEncoderConfig config;
7807 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
7808 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
7809 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
7810 config.simulcast_layers[i].active = true;
7811 }
7812 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007813 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01007814 "VP8", /*max qp*/ 56, /*screencast*/ false,
7815 /*screenshare enabled*/ false);
7816 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007817 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7818 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01007819
7820 // Capture a frame with all layers active.
7821 int64_t timestamp_ms = kFrameIntervalMs;
7822 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
7823 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7824 video_source_.IncomingCapturedFrame(
7825 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7826 WaitForEncodedFrame(timestamp_ms);
7827 // Expect encoded resolutions to match the expected simulcast layers.
7828 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7829 EXPECT_THAT(
7830 video_source_.sink_wants().resolutions,
7831 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
7832
7833 // Capture a frame with one of the layers inactive.
7834 timestamp_ms += kFrameIntervalMs;
7835 config.simulcast_layers[2].active = false;
7836 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
7837 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7838 video_source_.IncomingCapturedFrame(
7839 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7840 WaitForEncodedFrame(timestamp_ms);
7841
7842 // Expect encoded resolutions to match the expected simulcast layers.
7843 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7844 EXPECT_THAT(video_source_.sink_wants().resolutions,
7845 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
7846
7847 // Capture a frame with all but one layer turned off.
7848 timestamp_ms += kFrameIntervalMs;
7849 config.simulcast_layers[1].active = false;
7850 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
7851 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7852 video_source_.IncomingCapturedFrame(
7853 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7854 WaitForEncodedFrame(timestamp_ms);
7855
7856 // Expect encoded resolutions to match the expected simulcast layers.
7857 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7858 EXPECT_THAT(video_source_.sink_wants().resolutions,
7859 ::testing::ElementsAreArray({kLayer0Size}));
7860
7861 video_stream_encoder_->Stop();
7862}
7863
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007864TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007865 ResetEncoder("VP8", 1, 1, 1, false);
7866
Niels Möller8b692902021-06-14 12:04:57 +02007867 // Force encoder reconfig.
7868 video_source_.IncomingCapturedFrame(
7869 CreateFrame(1, codec_width_, codec_height_));
7870 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7871
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007872 // Set QP on encoded frame and pass the frame to encode complete callback.
7873 // Since QP is present QP parsing won't be triggered and the original value
7874 // should be kept.
7875 EncodedImage encoded_image;
7876 encoded_image.qp_ = 123;
7877 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7878 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7879 CodecSpecificInfo codec_info;
7880 codec_info.codecType = kVideoCodecVP8;
7881 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7882 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7883 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
7884 video_stream_encoder_->Stop();
7885}
7886
7887TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007888 ResetEncoder("VP8", 1, 1, 1, false);
7889
Niels Möller8b692902021-06-14 12:04:57 +02007890 // Force encoder reconfig.
7891 video_source_.IncomingCapturedFrame(
7892 CreateFrame(1, codec_width_, codec_height_));
7893 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7894
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007895 // Pass an encoded frame without QP to encode complete callback. QP should be
7896 // parsed and set.
7897 EncodedImage encoded_image;
7898 encoded_image.qp_ = -1;
7899 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7900 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7901 CodecSpecificInfo codec_info;
7902 codec_info.codecType = kVideoCodecVP8;
7903 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7904 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7905 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
7906 video_stream_encoder_->Stop();
7907}
7908
7909TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
7910 webrtc::test::ScopedFieldTrials field_trials(
7911 "WebRTC-QpParsingKillSwitch/Enabled/");
7912
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007913 ResetEncoder("VP8", 1, 1, 1, false);
7914
Niels Möller8b692902021-06-14 12:04:57 +02007915 // Force encoder reconfig.
7916 video_source_.IncomingCapturedFrame(
7917 CreateFrame(1, codec_width_, codec_height_));
7918 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7919
Sergey Silkin0e42cf72021-03-15 10:12:57 +01007920 EncodedImage encoded_image;
7921 encoded_image.qp_ = -1;
7922 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
7923 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
7924 CodecSpecificInfo codec_info;
7925 codec_info.codecType = kVideoCodecVP8;
7926 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
7927 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
7928 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
7929 video_stream_encoder_->Stop();
7930}
7931
Sergey Silkind19e3b92021-03-16 10:05:30 +00007932TEST_F(VideoStreamEncoderTest,
7933 QualityScalingNotAllowed_QualityScalingDisabled) {
7934 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
7935
7936 // Disable scaling settings in encoder info.
7937 fake_encoder_.SetQualityScaling(false);
7938 // Disable quality scaling in encoder config.
7939 video_encoder_config.is_quality_scaling_allowed = false;
7940 ConfigureEncoder(std::move(video_encoder_config));
7941
7942 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007943 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00007944
7945 test::FrameForwarder source;
7946 video_stream_encoder_->SetSource(
7947 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
7948 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
7949 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7950
7951 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
7952 WaitForEncodedFrame(1);
7953 video_stream_encoder_->TriggerQualityLow();
7954 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7955
7956 video_stream_encoder_->Stop();
7957}
7958
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08007959TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
7960 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
7961
7962 // Disable scaling settings in encoder info.
7963 fake_encoder_.SetQualityScaling(false);
7964 // Set QP trusted in encoder info.
7965 fake_encoder_.SetIsQpTrusted(true);
7966 // Enable quality scaling in encoder config.
7967 video_encoder_config.is_quality_scaling_allowed = false;
7968 ConfigureEncoder(std::move(video_encoder_config));
7969
7970 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007971 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08007972
7973 test::FrameForwarder source;
7974 video_stream_encoder_->SetSource(
7975 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
7976 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
7977 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7978
7979 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
7980 WaitForEncodedFrame(1);
7981 video_stream_encoder_->TriggerQualityLow();
7982 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
7983
7984 video_stream_encoder_->Stop();
7985}
7986
Shuhai Pengf2707702021-09-29 17:19:44 +08007987TEST_F(VideoStreamEncoderTest,
7988 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
7989 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
7990
7991 // Disable scaling settings in encoder info.
7992 fake_encoder_.SetQualityScaling(false);
7993 // Set QP trusted in encoder info.
7994 fake_encoder_.SetIsQpTrusted(true);
7995 // Enable quality scaling in encoder config.
7996 video_encoder_config.is_quality_scaling_allowed = false;
7997 ConfigureEncoder(std::move(video_encoder_config));
7998
7999 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008000 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008001
8002 test::FrameForwarder source;
8003 video_stream_encoder_->SetSource(
8004 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8005 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8006 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8007
8008 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8009 WaitForEncodedFrame(1);
8010 video_stream_encoder_->TriggerQualityLow();
8011 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8012
8013 video_stream_encoder_->Stop();
8014}
8015
8016TEST_F(VideoStreamEncoderTest,
8017 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8018 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8019
8020 // Disable scaling settings in encoder info.
8021 fake_encoder_.SetQualityScaling(false);
8022 // Set QP trusted in encoder info.
8023 fake_encoder_.SetIsQpTrusted(false);
8024 // Enable quality scaling in encoder config.
8025 video_encoder_config.is_quality_scaling_allowed = false;
8026 ConfigureEncoder(std::move(video_encoder_config));
8027
8028 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008029 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008030
8031 test::FrameForwarder source;
8032 video_stream_encoder_->SetSource(
8033 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8034 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8035 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8036
8037 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8038 WaitForEncodedFrame(1);
8039 video_stream_encoder_->TriggerQualityLow();
8040 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8041
8042 video_stream_encoder_->Stop();
8043}
8044
8045TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8046 // Set QP trusted in encoder info.
8047 fake_encoder_.SetIsQpTrusted(false);
8048
8049 const int MinEncBitrateKbps = 30;
8050 const int MaxEncBitrateKbps = 100;
8051 const int MinStartBitrateKbp = 50;
8052 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8053 /*frame_size_pixels=*/codec_width_ * codec_height_,
8054 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8055 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8056 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8057
8058 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008059 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008060
8061 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8062
8063 VideoEncoderConfig video_encoder_config;
8064 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8065 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8066 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8067 MinEncBitrateKbps * 1000;
8068 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8069 kMaxPayloadLength);
8070
8071 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8072 WaitForEncodedFrame(1);
8073 EXPECT_EQ(
8074 MaxEncBitrateKbps,
8075 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8076 EXPECT_EQ(
8077 MinEncBitrateKbps,
8078 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8079
8080 video_stream_encoder_->Stop();
8081}
8082
8083TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8084 // Set QP trusted in encoder info.
8085 fake_encoder_.SetIsQpTrusted(false);
8086
8087 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8088 EncoderInfoSettings::
8089 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8090 codec_width_ * codec_height_,
8091 EncoderInfoSettings::
8092 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8093 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8094
8095 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8096 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8097 const int TargetEncBitrate = MaxEncBitrate;
8098 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8099 DataRate::BitsPerSec(TargetEncBitrate),
8100 DataRate::BitsPerSec(TargetEncBitrate),
8101 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8102
8103 VideoEncoderConfig video_encoder_config;
8104 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8105 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8106 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8107 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8108 kMaxPayloadLength);
8109
8110 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8111 WaitForEncodedFrame(1);
8112 EXPECT_EQ(
8113 MaxEncBitrate / 1000,
8114 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8115 EXPECT_EQ(
8116 MinEncBitrate / 1000,
8117 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8118
8119 video_stream_encoder_->Stop();
8120}
8121
Sergey Silkind19e3b92021-03-16 10:05:30 +00008122#if !defined(WEBRTC_IOS)
8123// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8124// disabled by default on iOS.
8125TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8126 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8127
8128 // Disable scaling settings in encoder info.
8129 fake_encoder_.SetQualityScaling(false);
8130 // Enable quality scaling in encoder config.
8131 video_encoder_config.is_quality_scaling_allowed = true;
8132 ConfigureEncoder(std::move(video_encoder_config));
8133
8134 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008135 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008136
8137 test::FrameForwarder source;
8138 video_stream_encoder_->SetSource(
8139 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8140 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8141 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8142
8143 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8144 WaitForEncodedFrame(1);
8145 video_stream_encoder_->TriggerQualityLow();
8146 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8147
8148 video_stream_encoder_->Stop();
8149}
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008150
8151TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8152 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8153
8154 // Disable scaling settings in encoder info.
8155 fake_encoder_.SetQualityScaling(false);
8156 // Set QP trusted in encoder info.
8157 fake_encoder_.SetIsQpTrusted(true);
8158 // Enable quality scaling in encoder config.
8159 video_encoder_config.is_quality_scaling_allowed = true;
8160 ConfigureEncoder(std::move(video_encoder_config));
8161
8162 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008163 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008164
8165 test::FrameForwarder source;
8166 video_stream_encoder_->SetSource(
8167 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8168 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8169 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8170
8171 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8172 WaitForEncodedFrame(1);
8173 video_stream_encoder_->TriggerQualityLow();
8174 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8175
8176 video_stream_encoder_->Stop();
8177}
Shuhai Pengf2707702021-09-29 17:19:44 +08008178
8179TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8180 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8181
8182 // Disable scaling settings in encoder info.
8183 fake_encoder_.SetQualityScaling(false);
8184 // Set QP not trusted in encoder info.
8185 fake_encoder_.SetIsQpTrusted(false);
8186 // Enable quality scaling in encoder config.
8187 video_encoder_config.is_quality_scaling_allowed = true;
8188 ConfigureEncoder(std::move(video_encoder_config));
8189
8190 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008191 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008192
8193 test::FrameForwarder source;
8194 video_stream_encoder_->SetSource(
8195 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8196 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8197 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8198
8199 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8200 WaitForEncodedFrame(1);
8201 video_stream_encoder_->TriggerQualityLow();
8202 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8203 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8204 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8205
8206 video_stream_encoder_->Stop();
8207}
8208
8209TEST_F(VideoStreamEncoderTest,
8210 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8211 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8212
8213 // Disable scaling settings in encoder info.
8214 fake_encoder_.SetQualityScaling(false);
8215 // Set QP trusted in encoder info.
8216 fake_encoder_.SetIsQpTrusted(true);
8217 // Enable quality scaling in encoder config.
8218 video_encoder_config.is_quality_scaling_allowed = true;
8219 ConfigureEncoder(std::move(video_encoder_config));
8220
8221 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008222 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008223
8224 test::FrameForwarder source;
8225 video_stream_encoder_->SetSource(
8226 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8227 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8228 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8229
8230 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8231 WaitForEncodedFrame(1);
8232 video_stream_encoder_->TriggerQualityLow();
8233 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8234 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8235
8236 video_stream_encoder_->Stop();
8237}
8238
8239TEST_F(VideoStreamEncoderTest,
8240 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8241 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8242
8243 // Disable scaling settings in encoder info.
8244 fake_encoder_.SetQualityScaling(false);
8245 // Set QP trusted in encoder info.
8246 fake_encoder_.SetIsQpTrusted(false);
8247 // Enable quality scaling in encoder config.
8248 video_encoder_config.is_quality_scaling_allowed = true;
8249 ConfigureEncoder(std::move(video_encoder_config));
8250
8251 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008252 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008253
8254 test::FrameForwarder source;
8255 video_stream_encoder_->SetSource(
8256 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8257 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8258 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8259
8260 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8261 WaitForEncodedFrame(1);
8262 video_stream_encoder_->TriggerQualityLow();
8263 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8264
8265 video_stream_encoder_->Stop();
8266}
8267
Sergey Silkind19e3b92021-03-16 10:05:30 +00008268#endif
8269
Henrik Boström56db9ff2021-03-24 09:06:45 +01008270// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8271class VideoStreamEncoderWithRealEncoderTest
8272 : public VideoStreamEncoderTest,
8273 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8274 public:
8275 VideoStreamEncoderWithRealEncoderTest()
8276 : VideoStreamEncoderTest(),
8277 codec_type_(std::get<0>(GetParam())),
8278 allow_i420_conversion_(std::get<1>(GetParam())) {}
8279
8280 void SetUp() override {
8281 VideoStreamEncoderTest::SetUp();
8282 std::unique_ptr<VideoEncoder> encoder;
8283 switch (codec_type_) {
8284 case kVideoCodecVP8:
8285 encoder = VP8Encoder::Create();
8286 break;
8287 case kVideoCodecVP9:
8288 encoder = VP9Encoder::Create();
8289 break;
8290 case kVideoCodecAV1:
8291 encoder = CreateLibaomAv1Encoder();
8292 break;
8293 case kVideoCodecH264:
8294 encoder =
8295 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8296 break;
8297 case kVideoCodecMultiplex:
8298 mock_encoder_factory_for_multiplex_ =
8299 std::make_unique<MockVideoEncoderFactory>();
8300 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8301 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8302 .WillRepeatedly([] { return VP8Encoder::Create(); });
8303 encoder = std::make_unique<MultiplexEncoderAdapter>(
8304 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8305 false);
8306 break;
8307 default:
Artem Titovd3251962021-11-15 16:57:07 +01008308 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008309 }
8310 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8311 }
8312
8313 void TearDown() override {
8314 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 16:22:11 +02008315 // Ensure `video_stream_encoder_` is destroyed before
8316 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 09:06:45 +01008317 video_stream_encoder_.reset();
8318 VideoStreamEncoderTest::TearDown();
8319 }
8320
8321 protected:
8322 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8323 std::unique_ptr<VideoEncoder> encoder) {
8324 // Configure VSE to use the encoder.
8325 encoder_ = std::move(encoder);
8326 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8327 encoder_.get(), &encoder_selector_);
8328 video_send_config_.encoder_settings.encoder_factory =
8329 encoder_proxy_factory_.get();
8330 VideoEncoderConfig video_encoder_config;
8331 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8332 video_encoder_config_ = video_encoder_config.Copy();
8333 ConfigureEncoder(video_encoder_config_.Copy());
8334
8335 // Set bitrate to ensure frame is not dropped.
8336 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008337 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008338 }
8339
8340 const VideoCodecType codec_type_;
8341 const bool allow_i420_conversion_;
8342 NiceMock<MockEncoderSelector> encoder_selector_;
8343 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8344 std::unique_ptr<VideoEncoder> encoder_;
8345 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8346};
8347
8348TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8349 auto native_i420_frame = test::CreateMappableNativeFrame(
8350 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8351 video_source_.IncomingCapturedFrame(native_i420_frame);
8352 WaitForEncodedFrame(codec_width_, codec_height_);
8353
8354 auto mappable_native_buffer =
8355 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8356 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8357 mappable_native_buffer->GetMappedFramedBuffers();
8358 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8359 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8360 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8361 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8362}
8363
8364TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8365 auto native_nv12_frame = test::CreateMappableNativeFrame(
8366 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8367 video_source_.IncomingCapturedFrame(native_nv12_frame);
8368 WaitForEncodedFrame(codec_width_, codec_height_);
8369
8370 auto mappable_native_buffer =
8371 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8372 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8373 mappable_native_buffer->GetMappedFramedBuffers();
8374 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8375 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8376 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8377 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8378
8379 if (!allow_i420_conversion_) {
8380 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8381 }
8382}
8383
Erik Språng7444b192021-06-02 14:02:13 +02008384TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8385 if (codec_type_ == kVideoCodecMultiplex) {
8386 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8387 return;
8388 }
8389
8390 const size_t kNumSpatialLayers = 3u;
8391 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8392 const int kFrameWidth = 1280;
8393 const int kFrameHeight = 720;
8394 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8395 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8396 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8397 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8398 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8399 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8400
8401 VideoEncoderConfig config;
8402 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8403 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008404 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008405 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8406 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8407 vp9_settings.numberOfTemporalLayers = 3;
8408 vp9_settings.automaticResizeOn = false;
8409 config.encoder_specific_settings =
8410 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8411 vp9_settings);
8412 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8413 /*fps=*/30.0,
8414 /*first_active_layer=*/0,
8415 /*num_spatial_layers=*/3,
8416 /*num_temporal_layers=*/3,
8417 /*is_screenshare=*/false);
8418 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8419 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008420 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008421 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8422 /*fps=*/30.0,
8423 /*first_active_layer=*/0,
8424 /*num_spatial_layers=*/3,
8425 /*num_temporal_layers=*/3,
8426 /*is_screenshare=*/false);
8427 config.simulcast_layers[0].scalability_mode = "L3T3_KEY";
8428 } else {
8429 // Simulcast for VP8/H264.
8430 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8431 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8432 config.simulcast_layers[i].scale_resolution_down_by =
8433 kDownscaleFactors[i];
8434 config.simulcast_layers[i].active = true;
8435 }
8436 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8437 // Turn off frame dropping to prevent flakiness.
8438 VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings();
8439 h264_settings.frameDroppingOn = false;
8440 config.encoder_specific_settings = rtc::make_ref_counted<
8441 VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings);
8442 }
8443 }
8444
8445 auto set_layer_active = [&](int layer_idx, bool active) {
8446 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8447 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8448 config.spatial_layers[layer_idx].active = active;
8449 } else {
8450 config.simulcast_layers[layer_idx].active = active;
8451 }
8452 };
8453
8454 config.video_stream_factory =
8455 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8456 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8457 /*screencast*/ false,
8458 /*screenshare enabled*/ false);
8459 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008460 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8461 0, 0, 0);
Erik Språng7444b192021-06-02 14:02:13 +02008462
8463 // Capture a frame with all layers active.
8464 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8465 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8466 int64_t timestamp_ms = kFrameIntervalMs;
8467 video_source_.IncomingCapturedFrame(
8468 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8469
8470 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8471 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8472
8473 // Capture a frame with one of the layers inactive.
8474 set_layer_active(2, false);
8475 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8476 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8477 timestamp_ms += kFrameIntervalMs;
8478 video_source_.IncomingCapturedFrame(
8479 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8480 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8481
8482 // New target bitrates signaled based on lower resolution.
8483 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8484 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8485 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8486 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8487
8488 // Re-enable the top layer.
8489 set_layer_active(2, true);
8490 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8491 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8492 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8493
8494 // Bitrate target adjusted back up to enable HD layer...
8495 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8496 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8497 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8498 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8499
8500 // ...then add a new frame.
8501 timestamp_ms += kFrameIntervalMs;
8502 video_source_.IncomingCapturedFrame(
8503 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8504 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8505 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8506
8507 video_stream_encoder_->Stop();
8508}
8509
Henrik Boström56db9ff2021-03-24 09:06:45 +01008510std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8511 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8512 VideoCodecType codec_type = std::get<0>(info.param);
8513 bool allow_i420_conversion = std::get<1>(info.param);
8514 std::string str;
8515 switch (codec_type) {
8516 case kVideoCodecGeneric:
8517 str = "Generic";
8518 break;
8519 case kVideoCodecVP8:
8520 str = "VP8";
8521 break;
8522 case kVideoCodecVP9:
8523 str = "VP9";
8524 break;
8525 case kVideoCodecAV1:
8526 str = "AV1";
8527 break;
8528 case kVideoCodecH264:
8529 str = "H264";
8530 break;
8531 case kVideoCodecMultiplex:
8532 str = "Multiplex";
8533 break;
8534 default:
Artem Titovd3251962021-11-15 16:57:07 +01008535 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008536 }
8537 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8538 return str;
8539}
8540
8541constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8542 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8543constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8544 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8545constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
8546 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/true);
8547constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8548 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8549#if defined(WEBRTC_USE_H264)
8550constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8551 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8552
8553// The windows compiler does not tolerate #if statements inside the
8554// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8555// and without H264).
8556INSTANTIATE_TEST_SUITE_P(
8557 All,
8558 VideoStreamEncoderWithRealEncoderTest,
8559 ::testing::Values(kVP8DisallowConversion,
8560 kVP9DisallowConversion,
8561 kAV1AllowConversion,
8562 kMultiplexDisallowConversion,
8563 kH264AllowConversion),
8564 TestParametersVideoCodecAndAllowI420ConversionToString);
8565#else
8566INSTANTIATE_TEST_SUITE_P(
8567 All,
8568 VideoStreamEncoderWithRealEncoderTest,
8569 ::testing::Values(kVP8DisallowConversion,
8570 kVP9DisallowConversion,
8571 kAV1AllowConversion,
8572 kMultiplexDisallowConversion),
8573 TestParametersVideoCodecAndAllowI420ConversionToString);
8574#endif
8575
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008576class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8577 protected:
8578 void RunTest(const std::vector<VideoStream>& configs,
8579 const int expected_num_init_encode) {
8580 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008581 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008582 InsertFrameAndWaitForEncoded();
8583 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8584 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008585 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8586 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008587
8588 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8589 ConfigureEncoder(configs[1]);
8590 InsertFrameAndWaitForEncoded();
8591 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8592 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008593 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008594 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 10:07:11 +02008595 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008596
8597 video_stream_encoder_->Stop();
8598 }
8599
8600 void ConfigureEncoder(const VideoStream& stream) {
8601 VideoEncoderConfig config;
8602 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8603 config.max_bitrate_bps = stream.max_bitrate_bps;
8604 config.simulcast_layers[0] = stream;
8605 config.video_stream_factory =
8606 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8607 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8608 /*conference_mode=*/false);
8609 video_stream_encoder_->ConfigureEncoder(std::move(config),
8610 kMaxPayloadLength);
8611 }
8612
8613 void OnBitrateUpdated(DataRate bitrate) {
8614 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8615 bitrate, bitrate, bitrate, 0, 0, 0);
8616 }
8617
8618 void InsertFrameAndWaitForEncoded() {
8619 timestamp_ms_ += kFrameIntervalMs;
8620 video_source_.IncomingCapturedFrame(
8621 CreateFrame(timestamp_ms_, kWidth, kHeight));
8622 sink_.WaitForEncodedFrame(timestamp_ms_);
8623 }
8624
8625 void ExpectEqual(const VideoCodec& actual,
8626 const VideoStream& expected) const {
8627 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8628 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8629 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
8630 static_cast<unsigned int>(expected.min_bitrate_bps));
8631 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
8632 static_cast<unsigned int>(expected.max_bitrate_bps));
8633 EXPECT_EQ(actual.simulcastStream[0].width,
8634 kWidth / expected.scale_resolution_down_by);
8635 EXPECT_EQ(actual.simulcastStream[0].height,
8636 kHeight / expected.scale_resolution_down_by);
8637 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
8638 expected.num_temporal_layers);
8639 EXPECT_EQ(actual.ScalabilityMode(), expected.scalability_mode);
8640 }
8641
8642 VideoStream DefaultConfig() const {
8643 VideoStream stream;
8644 stream.max_framerate = 25;
8645 stream.min_bitrate_bps = 35000;
8646 stream.max_bitrate_bps = 900000;
8647 stream.scale_resolution_down_by = 1.0;
8648 stream.num_temporal_layers = 1;
8649 stream.bitrate_priority = 1.0;
8650 stream.scalability_mode = "";
8651 return stream;
8652 }
8653
8654 const int kWidth = 640;
8655 const int kHeight = 360;
8656 int64_t timestamp_ms_ = 0;
8657};
8658
8659TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
8660 VideoStream config1 = DefaultConfig();
8661 VideoStream config2 = config1;
8662 config2.max_framerate++;
8663
8664 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8665}
8666
8667TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
8668 VideoStream config1 = DefaultConfig();
8669 VideoStream config2 = config1;
8670 config2.min_bitrate_bps += 10000;
8671
8672 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8673}
8674
8675TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
8676 VideoStream config1 = DefaultConfig();
8677 VideoStream config2 = config1;
8678 config2.max_bitrate_bps += 100000;
8679
8680 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8681}
8682
8683TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
8684 VideoStream config1 = DefaultConfig();
8685 VideoStream config2 = config1;
8686 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
8687
8688 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8689}
8690
8691TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
8692 VideoStream config1 = DefaultConfig();
8693 VideoStream config2 = config1;
8694 config2.scale_resolution_down_by *= 2;
8695
8696 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8697}
8698
8699TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
8700 VideoStream config1 = DefaultConfig();
8701 VideoStream config2 = config1;
8702 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
8703
8704 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8705}
8706
8707TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
8708 VideoStream config1 = DefaultConfig();
8709 VideoStream config2 = config1;
8710 config2.scalability_mode = "L1T2";
8711
8712 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8713}
8714
Markus Handellb4e96d42021-11-05 12:00:55 +01008715TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
8716 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8717 auto* adapter_ptr = adapter.get();
8718 SimpleVideoStreamEncoderFactory factory;
8719 auto video_stream_encoder = factory.Create(std::move(adapter));
8720
8721 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(true));
8722 VideoEncoderConfig config;
8723 config.content_type = VideoEncoderConfig::ContentType::kScreen;
8724 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
Markus Handell9a478b52021-11-18 16:07:01 +01008725 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01008726 Mock::VerifyAndClearExpectations(adapter_ptr);
8727
8728 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(false));
8729 VideoEncoderConfig config2;
8730 config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
8731 video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
Markus Handell9a478b52021-11-18 16:07:01 +01008732 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01008733}
8734
8735TEST(VideoStreamEncoderFrameCadenceTest,
8736 ForwardsFramesIntoFrameCadenceAdapter) {
8737 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8738 auto* adapter_ptr = adapter.get();
8739 test::FrameForwarder video_source;
8740 SimpleVideoStreamEncoderFactory factory;
8741 auto video_stream_encoder = factory.Create(std::move(adapter));
8742 video_stream_encoder->SetSource(
8743 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8744
8745 EXPECT_CALL(*adapter_ptr, OnFrame);
8746 auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
8747 video_source.IncomingCapturedFrame(
8748 VideoFrame::Builder()
8749 .set_video_frame_buffer(std::move(buffer))
8750 .set_ntp_time_ms(0)
8751 .set_timestamp_ms(0)
8752 .set_rotation(kVideoRotation_0)
8753 .build());
8754}
8755
Markus Handellee225432021-11-29 12:35:12 +01008756TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) {
8757 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8758 auto* adapter_ptr = adapter.get();
8759 test::FrameForwarder video_source;
8760 SimpleVideoStreamEncoderFactory factory;
8761 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8762 nullptr;
8763 EXPECT_CALL(*adapter_ptr, Initialize)
8764 .WillOnce(Invoke([&video_stream_encoder_callback](
8765 FrameCadenceAdapterInterface::Callback* callback) {
8766 video_stream_encoder_callback = callback;
8767 }));
8768 TaskQueueBase* encoder_queue = nullptr;
8769 auto video_stream_encoder =
8770 factory.Create(std::move(adapter), &encoder_queue);
8771
8772 // This is just to make the VSE operational. We'll feed a frame directly by
8773 // the callback interface.
8774 video_stream_encoder->SetSource(
8775 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8776
8777 VideoEncoderConfig video_encoder_config;
8778 test::FillEncoderConfiguration(kVideoCodecGeneric, 1, &video_encoder_config);
8779 video_stream_encoder->ConfigureEncoder(std::move(video_encoder_config),
8780 /*max_data_payload_length=*/1000);
8781
8782 EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps);
8783 EXPECT_CALL(*adapter_ptr, UpdateFrameRate);
8784 encoder_queue->PostTask(ToQueuedTask([video_stream_encoder_callback] {
8785 video_stream_encoder_callback->OnFrame(
8786 Timestamp::Millis(1), 1,
8787 VideoFrame::Builder()
8788 .set_video_frame_buffer(
8789 rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16))
8790 .set_ntp_time_ms(0)
8791 .set_timestamp_ms(0)
8792 .set_rotation(kVideoRotation_0)
8793 .build());
8794 }));
8795 factory.DepleteTaskQueues();
8796}
8797
perkj26091b12016-09-01 01:17:40 -07008798} // namespace webrtc