blob: aea1988df9f769989ede99db2f54a641222acf6e [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Markus Handell9a478b52021-11-18 16:07:01 +010020#include "api/rtp_parameters.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020021#include "api/task_queue/default_task_queue_factory.h"
Markus Handell818e7fb2021-12-30 13:01:33 +010022#include "api/task_queue/task_queue_base.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010023#include "api/task_queue/task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020024#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010025#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010026#include "api/test/mock_video_encoder_factory.h"
Markus Handell8d87c462021-12-16 11:37:16 +010027#include "api/units/data_rate.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080028#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020029#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020030#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020031#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010032#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010033#include "api/video_codecs/sdp_video_format.h"
Elad Alon370f93a2019-06-11 14:57:57 +020034#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020035#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010036#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020037#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010038#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020039#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070040#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080041#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020042#include "media/engine/webrtc_video_engine.h"
philipel95701502022-01-18 18:47:52 +010043#include "modules/video_coding/codecs/av1/libaom_av1_encoder_supported.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010044#include "modules/video_coding/codecs/h264/include/h264.h"
45#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
46#include "modules/video_coding/codecs/vp8/include/vp8.h"
47#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020048#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020049#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020050#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010051#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020052#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010053#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020054#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020055#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080056#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020057#include "rtc_base/synchronization/mutex.h"
Erik Språng7ca375c2019-02-06 16:20:17 +010058#include "system_wrappers/include/field_trial.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020059#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020060#include "test/encoder_settings.h"
61#include "test/fake_encoder.h"
Kári Tristan Helgason639602a2018-08-02 10:51:40 +020062#include "test/field_trial.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010063#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020064#include "test/gmock.h"
65#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010066#include "test/mappable_native_buffer.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020067#include "test/time_controller/simulated_time_controller.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020068#include "test/video_encoder_proxy_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010069#include "video/frame_cadence_adapter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020070#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070071
72namespace webrtc {
73
sprang57c2fff2017-01-16 06:24:02 -080074using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020075using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020076using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020077using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020078using ::testing::Ge;
79using ::testing::Gt;
Markus Handellee225432021-11-29 12:35:12 +010080using ::testing::Invoke;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020081using ::testing::Le;
82using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010083using ::testing::Matcher;
Markus Handellb4e96d42021-11-05 12:00:55 +010084using ::testing::Mock;
philipel9b058032020-02-10 11:30:00 +010085using ::testing::NiceMock;
Markus Handellb4e96d42021-11-05 12:00:55 +010086using ::testing::Optional;
philipel9b058032020-02-10 11:30:00 +010087using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010088using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020089using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080090
perkj803d97f2016-11-01 11:45:46 -070091namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020092const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010093const int kQpLow = 1;
94const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020095const int kMinFramerateFps = 2;
96const int kMinBalancedFramerateFps = 7;
97const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080098const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 10:07:11 +020099const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
100const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
101const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
102const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 07:02:22 -0800103const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -0700104const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200105const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +0200106const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200107const VideoEncoder::ResolutionBitrateLimits
108 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
109const VideoEncoder::ResolutionBitrateLimits
110 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800111
Asa Persson606d3cb2021-10-04 10:07:11 +0200112uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200113 0x00, 0x00, 0x03, 0x03, 0xF4,
114 0x05, 0x03, 0xC7, 0xE0, 0x1B,
115 0x41, 0x10, 0x8D, 0x00};
116
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100117const uint8_t kCodedFrameVp8Qp25[] = {
118 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
119 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
120 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
121
Markus Handell818e7fb2021-12-30 13:01:33 +0100122VideoFrame CreateSimpleNV12Frame() {
123 return VideoFrame::Builder()
124 .set_video_frame_buffer(rtc::make_ref_counted<NV12Buffer>(
125 /*width=*/16, /*height=*/16))
126 .build();
127}
128
Markus Handell8d87c462021-12-16 11:37:16 +0100129void PassAFrame(
130 TaskQueueBase* encoder_queue,
131 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback,
132 int64_t ntp_time_ms) {
133 encoder_queue->PostTask(
134 ToQueuedTask([video_stream_encoder_callback, ntp_time_ms] {
Markus Handell818e7fb2021-12-30 13:01:33 +0100135 video_stream_encoder_callback->OnFrame(Timestamp::Millis(ntp_time_ms),
136 1, CreateSimpleNV12Frame());
Markus Handell8d87c462021-12-16 11:37:16 +0100137 }));
138}
139
perkj803d97f2016-11-01 11:45:46 -0700140class TestBuffer : public webrtc::I420Buffer {
141 public:
142 TestBuffer(rtc::Event* event, int width, int height)
143 : I420Buffer(width, height), event_(event) {}
144
145 private:
146 friend class rtc::RefCountedObject<TestBuffer>;
147 ~TestBuffer() override {
148 if (event_)
149 event_->Set();
150 }
151 rtc::Event* const event_;
152};
153
Henrik Boström56db9ff2021-03-24 09:06:45 +0100154// A fake native buffer that can't be converted to I420. Upon scaling, it
155// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700156class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
157 public:
158 FakeNativeBuffer(rtc::Event* event, int width, int height)
159 : event_(event), width_(width), height_(height) {}
160 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
161 int width() const override { return width_; }
162 int height() const override { return height_; }
163 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
164 return nullptr;
165 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100166 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
167 int offset_x,
168 int offset_y,
169 int crop_width,
170 int crop_height,
171 int scaled_width,
172 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200173 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
174 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100175 }
Noah Richards51db4212019-06-12 06:59:12 -0700176
177 private:
178 friend class rtc::RefCountedObject<FakeNativeBuffer>;
179 ~FakeNativeBuffer() override {
180 if (event_)
181 event_->Set();
182 }
183 rtc::Event* const event_;
184 const int width_;
185 const int height_;
186};
187
Evan Shrubsole895556e2020-10-05 09:15:13 +0200188// A fake native buffer that is backed by an NV12 buffer.
189class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
190 public:
191 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
192 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
193
194 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
195 int width() const override { return nv12_buffer_->width(); }
196 int height() const override { return nv12_buffer_->height(); }
197 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
198 return nv12_buffer_->ToI420();
199 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200200 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
201 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
202 if (absl::c_find(types, Type::kNV12) != types.end()) {
203 return nv12_buffer_;
204 }
205 return nullptr;
206 }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200207 const NV12BufferInterface* GetNV12() const { return nv12_buffer_; }
208
209 private:
210 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
211 ~FakeNV12NativeBuffer() override {
212 if (event_)
213 event_->Set();
214 }
215 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
216 rtc::Event* const event_;
217};
218
Niels Möller7dc26b72017-12-06 10:27:48 +0100219class CpuOveruseDetectorProxy : public OveruseFrameDetector {
220 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200221 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
222 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200223 last_target_framerate_fps_(-1),
224 framerate_updated_event_(true /* manual_reset */,
225 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100226 virtual ~CpuOveruseDetectorProxy() {}
227
228 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200229 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100230 last_target_framerate_fps_ = framerate_fps;
231 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200232 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100233 }
234
235 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200236 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100237 return last_target_framerate_fps_;
238 }
239
Niels Möller4db138e2018-04-19 09:04:13 +0200240 CpuOveruseOptions GetOptions() { return options_; }
241
Henrik Boström381d1092020-05-12 18:49:07 +0200242 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
243
Niels Möller7dc26b72017-12-06 10:27:48 +0100244 private:
Markus Handella3765182020-07-08 13:13:32 +0200245 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100246 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200247 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100248};
249
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200250class FakeVideoSourceRestrictionsListener
251 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200252 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200253 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200254 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200255 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200256 RTC_DCHECK(was_restrictions_updated_);
257 }
258
259 rtc::Event* restrictions_updated_event() {
260 return &restrictions_updated_event_;
261 }
262
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200263 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200264 void OnVideoSourceRestrictionsUpdated(
265 VideoSourceRestrictions restrictions,
266 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200267 rtc::scoped_refptr<Resource> reason,
268 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200269 was_restrictions_updated_ = true;
270 restrictions_updated_event_.Set();
271 }
272
273 private:
274 bool was_restrictions_updated_;
275 rtc::Event restrictions_updated_event_;
276};
277
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200278auto WantsFps(Matcher<int> fps_matcher) {
279 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
280 fps_matcher);
281}
282
283auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
284 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
285 AllOf(max_pixel_matcher, Gt(0)));
286}
287
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200288auto ResolutionMax() {
289 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200290 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200291 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
292 Eq(absl::nullopt)));
293}
294
295auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200296 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200297}
298
299auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200300 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200301}
302
303auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200304 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200305}
306
307auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200308 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200309}
310
311auto FpsMaxResolutionMax() {
312 return AllOf(FpsMax(), ResolutionMax());
313}
314
315auto UnlimitedSinkWants() {
316 return AllOf(FpsUnlimited(), ResolutionMax());
317}
318
319auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
320 Matcher<int> fps_range_matcher;
321
322 if (last_frame_pixels <= 320 * 240) {
323 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200324 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200325 fps_range_matcher = AllOf(Ge(10), Le(15));
326 } else if (last_frame_pixels <= 640 * 480) {
327 fps_range_matcher = Ge(15);
328 } else {
329 fps_range_matcher = Eq(kDefaultFramerate);
330 }
331 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
332 fps_range_matcher);
333}
334
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200335auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
336 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
337 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
338}
339
340auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
341 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
342}
343
344auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
345 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
346}
347
348auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
349 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
350 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
351}
352
353auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
354 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
355 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
356}
357
358auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
359 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
360 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
361}
362
363auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
364 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
365 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
366}
367
mflodmancc3d4422017-08-03 08:27:51 -0700368class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700369 public:
Markus Handell9a478b52021-11-18 16:07:01 +0100370 VideoStreamEncoderUnderTest(
371 TimeController* time_controller,
372 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter,
373 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
374 encoder_queue,
375 SendStatisticsProxy* stats_proxy,
376 const VideoStreamEncoderSettings& settings,
377 VideoStreamEncoder::BitrateAllocationCallbackType
378 allocation_callback_type)
379 : VideoStreamEncoder(time_controller->GetClock(),
380 1 /* number_of_cores */,
381 stats_proxy,
382 settings,
383 std::unique_ptr<OveruseFrameDetector>(
384 overuse_detector_proxy_ =
385 new CpuOveruseDetectorProxy(stats_proxy)),
386 std::move(cadence_adapter),
387 std::move(encoder_queue),
388 allocation_callback_type),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200389 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200390 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200391 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200392 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200393 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200394 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200395 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200396 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100397 }
perkj803d97f2016-11-01 11:45:46 -0700398
Henrik Boström381d1092020-05-12 18:49:07 +0200399 void SetSourceAndWaitForRestrictionsUpdated(
400 rtc::VideoSourceInterface<VideoFrame>* source,
401 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200402 FakeVideoSourceRestrictionsListener listener;
403 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200404 SetSource(source, degradation_preference);
405 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200406 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200407 }
408
409 void SetSourceAndWaitForFramerateUpdated(
410 rtc::VideoSourceInterface<VideoFrame>* source,
411 const DegradationPreference& degradation_preference) {
412 overuse_detector_proxy_->framerate_updated_event()->Reset();
413 SetSource(source, degradation_preference);
414 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
415 }
416
417 void OnBitrateUpdatedAndWaitForManagedResources(
418 DataRate target_bitrate,
419 DataRate stable_target_bitrate,
420 DataRate link_allocation,
421 uint8_t fraction_lost,
422 int64_t round_trip_time_ms,
423 double cwnd_reduce_ratio) {
424 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
425 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
426 // Bitrate is updated on the encoder queue.
427 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200428 }
429
kthelgason2fc52542017-03-03 00:24:41 -0800430 // This is used as a synchronisation mechanism, to make sure that the
431 // encoder queue is not blocked before we start sending it frames.
432 void WaitUntilTaskQueueIsIdle() {
Markus Handell28c71802021-11-08 10:11:55 +0100433 time_controller_->AdvanceTime(TimeDelta::Zero());
kthelgason2fc52542017-03-03 00:24:41 -0800434 }
435
Henrik Boström91aa7322020-04-28 12:24:33 +0200436 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200437 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200438 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200439 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200440 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200441 event.Set();
442 });
443 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100444 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200445 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200446
Henrik Boström91aa7322020-04-28 12:24:33 +0200447 void TriggerCpuUnderuse() {
448 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200449 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200450 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200451 event.Set();
452 });
453 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100454 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200455 }
kthelgason876222f2016-11-29 01:44:11 -0800456
Henrik Boström91aa7322020-04-28 12:24:33 +0200457 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200458 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200459 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200460 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200461 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200462 event.Set();
463 });
464 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100465 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200466 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200467 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200468 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200469 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200470 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200471 event.Set();
472 });
473 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100474 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200475 }
476
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200477 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100478 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200479 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
480 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200481 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700482};
483
Noah Richards51db4212019-06-12 06:59:12 -0700484// Simulates simulcast behavior and makes highest stream resolutions divisible
485// by 4.
486class CroppingVideoStreamFactory
487 : public VideoEncoderConfig::VideoStreamFactoryInterface {
488 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200489 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700490
491 private:
492 std::vector<VideoStream> CreateEncoderStreams(
493 int width,
494 int height,
495 const VideoEncoderConfig& encoder_config) override {
496 std::vector<VideoStream> streams = test::CreateVideoStreams(
497 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700498 return streams;
499 }
Noah Richards51db4212019-06-12 06:59:12 -0700500};
501
sprangb1ca0732017-02-01 08:38:12 -0800502class AdaptingFrameForwarder : public test::FrameForwarder {
503 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200504 explicit AdaptingFrameForwarder(TimeController* time_controller)
505 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700506 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800507
508 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200509 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800510 adaptation_enabled_ = enabled;
511 }
512
asaperssonfab67072017-04-04 05:51:49 -0700513 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200514 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800515 return adaptation_enabled_;
516 }
517
Henrik Boström1124ed12021-02-25 10:30:39 +0100518 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
519 // the resolution or frame rate was different than it is currently. If
520 // something else is modified, such as encoder resolutions, but the resolution
521 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700522 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200523 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700524 return last_wants_;
525 }
526
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200527 absl::optional<int> last_sent_width() const { return last_width_; }
528 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800529
sprangb1ca0732017-02-01 08:38:12 -0800530 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200531 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +0100532 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200533
sprangb1ca0732017-02-01 08:38:12 -0800534 int cropped_width = 0;
535 int cropped_height = 0;
536 int out_width = 0;
537 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700538 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000539 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
540 << "w=" << video_frame.width()
541 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700542 if (adapter_.AdaptFrameResolution(
543 video_frame.width(), video_frame.height(),
544 video_frame.timestamp_us() * 1000, &cropped_width,
545 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100546 VideoFrame adapted_frame =
547 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200548 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100549 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200550 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100551 .set_timestamp_ms(99)
552 .set_rotation(kVideoRotation_0)
553 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100554 if (video_frame.has_update_rect()) {
555 adapted_frame.set_update_rect(
556 video_frame.update_rect().ScaleWithFrame(
557 video_frame.width(), video_frame.height(), 0, 0,
558 video_frame.width(), video_frame.height(), out_width,
559 out_height));
560 }
sprangc5d62e22017-04-02 23:53:04 -0700561 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800562 last_width_.emplace(adapted_frame.width());
563 last_height_.emplace(adapted_frame.height());
564 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200565 last_width_ = absl::nullopt;
566 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700567 }
sprangb1ca0732017-02-01 08:38:12 -0800568 } else {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000569 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800570 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800571 last_width_.emplace(video_frame.width());
572 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800573 }
574 }
575
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200576 void OnOutputFormatRequest(int width, int height) {
577 absl::optional<std::pair<int, int>> target_aspect_ratio =
578 std::make_pair(width, height);
579 absl::optional<int> max_pixel_count = width * height;
580 absl::optional<int> max_fps;
581 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
582 max_fps);
583 }
584
sprangb1ca0732017-02-01 08:38:12 -0800585 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
586 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200587 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100588 rtc::VideoSinkWants prev_wants = sink_wants_locked();
589 bool did_adapt =
590 prev_wants.max_pixel_count != wants.max_pixel_count ||
591 prev_wants.target_pixel_count != wants.target_pixel_count ||
592 prev_wants.max_framerate_fps != wants.max_framerate_fps;
593 if (did_adapt) {
594 last_wants_ = prev_wants;
595 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100596 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200597 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800598 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200599
600 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800601 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200602 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
603 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200604 absl::optional<int> last_width_;
605 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800606};
sprangc5d62e22017-04-02 23:53:04 -0700607
Niels Möller213618e2018-07-24 09:29:58 +0200608// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700609class MockableSendStatisticsProxy : public SendStatisticsProxy {
610 public:
611 MockableSendStatisticsProxy(Clock* clock,
612 const VideoSendStream::Config& config,
613 VideoEncoderConfig::ContentType content_type)
614 : SendStatisticsProxy(clock, config, content_type) {}
615
616 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200617 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700618 if (mock_stats_)
619 return *mock_stats_;
620 return SendStatisticsProxy::GetStats();
621 }
622
Niels Möller213618e2018-07-24 09:29:58 +0200623 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200624 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200625 if (mock_stats_)
626 return mock_stats_->input_frame_rate;
627 return SendStatisticsProxy::GetInputFrameRate();
628 }
sprangc5d62e22017-04-02 23:53:04 -0700629 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200630 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700631 mock_stats_.emplace(stats);
632 }
633
634 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200635 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700636 mock_stats_.reset();
637 }
638
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200639 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
640 on_frame_dropped_ = std::move(callback);
641 }
642
sprangc5d62e22017-04-02 23:53:04 -0700643 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200644 void OnFrameDropped(DropReason reason) override {
645 SendStatisticsProxy::OnFrameDropped(reason);
646 if (on_frame_dropped_)
647 on_frame_dropped_(reason);
648 }
649
Markus Handella3765182020-07-08 13:13:32 +0200650 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200651 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200652 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700653};
654
Markus Handellb4e96d42021-11-05 12:00:55 +0100655class SimpleVideoStreamEncoderFactory {
656 public:
657 class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
658 public:
659 using VideoStreamEncoder::VideoStreamEncoder;
660 ~AdaptedVideoStreamEncoder() { Stop(); }
661 };
662
Markus Handell8d87c462021-12-16 11:37:16 +0100663 class MockFakeEncoder : public test::FakeEncoder {
664 public:
665 using FakeEncoder::FakeEncoder;
666 MOCK_METHOD(CodecSpecificInfo,
667 EncodeHook,
668 (EncodedImage & encoded_image,
669 rtc::scoped_refptr<EncodedImageBuffer> buffer),
670 (override));
671 };
672
Markus Handellee225432021-11-29 12:35:12 +0100673 SimpleVideoStreamEncoderFactory() {
Markus Handellb4e96d42021-11-05 12:00:55 +0100674 encoder_settings_.encoder_factory = &encoder_factory_;
Markus Handellee225432021-11-29 12:35:12 +0100675 encoder_settings_.bitrate_allocator_factory =
676 bitrate_allocator_factory_.get();
Markus Handellb4e96d42021-11-05 12:00:55 +0100677 }
678
Markus Handell818e7fb2021-12-30 13:01:33 +0100679 std::unique_ptr<AdaptedVideoStreamEncoder> CreateWithEncoderQueue(
Markus Handellee225432021-11-29 12:35:12 +0100680 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
Markus Handell818e7fb2021-12-30 13:01:33 +0100681 std::unique_ptr<TaskQueueBase, TaskQueueDeleter> encoder_queue) {
Markus Handellb4e96d42021-11-05 12:00:55 +0100682 auto result = std::make_unique<AdaptedVideoStreamEncoder>(
683 time_controller_.GetClock(),
684 /*number_of_cores=*/1,
685 /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
686 std::make_unique<CpuOveruseDetectorProxy>(/*stats_proxy=*/nullptr),
Markus Handellee225432021-11-29 12:35:12 +0100687 std::move(zero_hertz_adapter), std::move(encoder_queue),
Markus Handellb4e96d42021-11-05 12:00:55 +0100688 VideoStreamEncoder::BitrateAllocationCallbackType::
689 kVideoBitrateAllocation);
690 result->SetSink(&sink_, /*rotation_applied=*/false);
691 return result;
692 }
693
Markus Handell818e7fb2021-12-30 13:01:33 +0100694 std::unique_ptr<AdaptedVideoStreamEncoder> Create(
695 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
696 TaskQueueBase** encoder_queue_ptr = nullptr) {
697 auto encoder_queue =
698 time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
699 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
700 if (encoder_queue_ptr)
701 *encoder_queue_ptr = encoder_queue.get();
702 return CreateWithEncoderQueue(std::move(zero_hertz_adapter),
703 std::move(encoder_queue));
704 }
705
Markus Handell9a478b52021-11-18 16:07:01 +0100706 void DepleteTaskQueues() { time_controller_.AdvanceTime(TimeDelta::Zero()); }
Markus Handell8d87c462021-12-16 11:37:16 +0100707 MockFakeEncoder& GetMockFakeEncoder() { return mock_fake_encoder_; }
Markus Handell9a478b52021-11-18 16:07:01 +0100708
Markus Handell818e7fb2021-12-30 13:01:33 +0100709 GlobalSimulatedTimeController* GetTimeController() {
710 return &time_controller_;
711 }
712
Markus Handellb4e96d42021-11-05 12:00:55 +0100713 private:
714 class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
715 public:
716 ~NullEncoderSink() override = default;
717 void OnEncoderConfigurationChanged(
718 std::vector<VideoStream> streams,
719 bool is_svc,
720 VideoEncoderConfig::ContentType content_type,
721 int min_transmit_bitrate_bps) override {}
722 void OnBitrateAllocationUpdated(
723 const VideoBitrateAllocation& allocation) override {}
724 void OnVideoLayersAllocationUpdated(
725 VideoLayersAllocation allocation) override {}
726 Result OnEncodedImage(
727 const EncodedImage& encoded_image,
728 const CodecSpecificInfo* codec_specific_info) override {
729 return Result(EncodedImageCallback::Result::OK);
730 }
731 };
732
Markus Handellee225432021-11-29 12:35:12 +0100733 GlobalSimulatedTimeController time_controller_{Timestamp::Millis(0)};
734 std::unique_ptr<TaskQueueFactory> task_queue_factory_{
735 time_controller_.CreateTaskQueueFactory()};
736 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_ =
737 std::make_unique<MockableSendStatisticsProxy>(
738 time_controller_.GetClock(),
739 VideoSendStream::Config(nullptr),
740 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo);
741 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_ =
742 CreateBuiltinVideoBitrateAllocatorFactory();
743 VideoStreamEncoderSettings encoder_settings_{
744 VideoEncoder::Capabilities(/*loss_notification=*/false)};
Markus Handell8d87c462021-12-16 11:37:16 +0100745 MockFakeEncoder mock_fake_encoder_{time_controller_.GetClock()};
746 test::VideoEncoderProxyFactory encoder_factory_{&mock_fake_encoder_};
Markus Handellb4e96d42021-11-05 12:00:55 +0100747 NullEncoderSink sink_;
748};
749
750class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
751 public:
752 MOCK_METHOD(void, Initialize, (Callback * callback), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100753 MOCK_METHOD(void,
754 SetZeroHertzModeEnabled,
755 (absl::optional<ZeroHertzModeParams>),
756 (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100757 MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
Markus Handellee225432021-11-29 12:35:12 +0100758 MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override));
759 MOCK_METHOD(void, UpdateFrameRate, (), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100760 MOCK_METHOD(void,
761 UpdateLayerQualityConvergence,
762 (int spatial_index, bool converged),
763 (override));
764 MOCK_METHOD(void,
765 UpdateLayerStatus,
766 (int spatial_index, bool enabled),
767 (override));
Markus Handell818e7fb2021-12-30 13:01:33 +0100768 MOCK_METHOD(void, ProcessKeyFrameRequest, (), (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100769};
770
philipel9b058032020-02-10 11:30:00 +0100771class MockEncoderSelector
772 : public VideoEncoderFactory::EncoderSelectorInterface {
773 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200774 MOCK_METHOD(void,
775 OnCurrentEncoder,
776 (const SdpVideoFormat& format),
777 (override));
778 MOCK_METHOD(absl::optional<SdpVideoFormat>,
779 OnAvailableBitrate,
780 (const DataRate& rate),
781 (override));
782 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100783};
784
Markus Handell2e0f4f02021-12-21 19:14:58 +0100785class MockVideoSourceInterface : public rtc::VideoSourceInterface<VideoFrame> {
786 public:
787 MOCK_METHOD(void,
788 AddOrUpdateSink,
789 (rtc::VideoSinkInterface<VideoFrame>*,
790 const rtc::VideoSinkWants&),
791 (override));
792 MOCK_METHOD(void,
793 RemoveSink,
794 (rtc::VideoSinkInterface<VideoFrame>*),
795 (override));
796 MOCK_METHOD(void, RequestRefreshFrame, (), (override));
797};
798
perkj803d97f2016-11-01 11:45:46 -0700799} // namespace
800
mflodmancc3d4422017-08-03 08:27:51 -0700801class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700802 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200803 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700804
mflodmancc3d4422017-08-03 08:27:51 -0700805 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700806 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700807 codec_width_(320),
808 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200809 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200810 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200811 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700812 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200813 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700814 video_send_config_,
815 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200816 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700817
818 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700819 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700820 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200821 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800822 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200823 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200824 video_send_config_.rtp.payload_name = "FAKE";
825 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700826
Per512ecb32016-09-23 15:52:06 +0200827 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200828 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200829 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
830 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
831 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100832 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700833
Niels Möllerf1338562018-04-26 09:51:47 +0200834 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800835 }
836
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100837 void ConfigureEncoder(
838 VideoEncoderConfig video_encoder_config,
839 VideoStreamEncoder::BitrateAllocationCallbackType
840 allocation_callback_type =
841 VideoStreamEncoder::BitrateAllocationCallbackType::
842 kVideoBitrateAllocationWhenScreenSharing) {
mflodmancc3d4422017-08-03 08:27:51 -0700843 if (video_stream_encoder_)
844 video_stream_encoder_->Stop();
Markus Handell9a478b52021-11-18 16:07:01 +0100845
846 auto encoder_queue = GetTaskQueueFactory()->CreateTaskQueue(
847 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
848 TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
849 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter =
850 FrameCadenceAdapterInterface::Create(time_controller_.GetClock(),
851 encoder_queue_ptr);
852 video_stream_encoder_ = std::make_unique<VideoStreamEncoderUnderTest>(
853 &time_controller_, std::move(cadence_adapter), std::move(encoder_queue),
854 stats_proxy_.get(), video_send_config_.encoder_settings,
855 allocation_callback_type);
Asa Persson606d3cb2021-10-04 10:07:11 +0200856 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 08:27:51 -0700857 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700858 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 10:07:11 +0200859 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 08:27:51 -0700860 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200861 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700862 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800863 }
864
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100865 void ResetEncoder(const std::string& payload_name,
866 size_t num_streams,
867 size_t num_temporal_layers,
868 unsigned char num_spatial_layers,
869 bool screenshare,
870 VideoStreamEncoder::BitrateAllocationCallbackType
871 allocation_callback_type =
872 VideoStreamEncoder::BitrateAllocationCallbackType::
873 kVideoBitrateAllocationWhenScreenSharing) {
Niels Möller259a4972018-04-05 15:36:51 +0200874 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800875
876 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200877 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
878 num_streams, &video_encoder_config);
879 for (auto& layer : video_encoder_config.simulcast_layers) {
880 layer.num_temporal_layers = num_temporal_layers;
881 layer.max_framerate = kDefaultFramerate;
882 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100883 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 10:07:11 +0200884 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 07:06:52 -0700885 video_encoder_config.content_type =
886 screenshare ? VideoEncoderConfig::ContentType::kScreen
887 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700888 if (payload_name == "VP9") {
889 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
890 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200891 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700892 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200893 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
894 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700895 }
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100896 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type);
perkj26091b12016-09-01 01:17:40 -0700897 }
898
sprang57c2fff2017-01-16 06:24:02 -0800899 VideoFrame CreateFrame(int64_t ntp_time_ms,
900 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200901 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200902 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200903 destruction_event, codec_width_, codec_height_))
904 .set_ntp_time_ms(ntp_time_ms)
905 .set_timestamp_ms(99)
906 .set_rotation(kVideoRotation_0)
907 .build();
perkj26091b12016-09-01 01:17:40 -0700908 }
909
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100910 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
911 rtc::Event* destruction_event,
912 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200913 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200914 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200915 destruction_event, codec_width_, codec_height_))
916 .set_ntp_time_ms(ntp_time_ms)
917 .set_timestamp_ms(99)
918 .set_rotation(kVideoRotation_0)
919 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
920 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100921 }
922
sprang57c2fff2017-01-16 06:24:02 -0800923 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200924 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
925 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200926 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200927 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200928 .set_ntp_time_ms(ntp_time_ms)
929 .set_timestamp_ms(ntp_time_ms)
930 .set_rotation(kVideoRotation_0)
931 .build();
perkj803d97f2016-11-01 11:45:46 -0700932 }
933
Evan Shrubsole895556e2020-10-05 09:15:13 +0200934 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200935 return VideoFrame::Builder()
936 .set_video_frame_buffer(NV12Buffer::Create(width, height))
937 .set_ntp_time_ms(ntp_time_ms)
938 .set_timestamp_ms(ntp_time_ms)
939 .set_rotation(kVideoRotation_0)
940 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200941 }
942
Noah Richards51db4212019-06-12 06:59:12 -0700943 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
944 rtc::Event* destruction_event,
945 int width,
946 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200947 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200948 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200949 destruction_event, width, height))
950 .set_ntp_time_ms(ntp_time_ms)
951 .set_timestamp_ms(99)
952 .set_rotation(kVideoRotation_0)
953 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700954 }
955
Evan Shrubsole895556e2020-10-05 09:15:13 +0200956 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
957 rtc::Event* destruction_event,
958 int width,
959 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200960 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200961 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200962 destruction_event, width, height))
963 .set_ntp_time_ms(ntp_time_ms)
964 .set_timestamp_ms(99)
965 .set_rotation(kVideoRotation_0)
966 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200967 }
968
Noah Richards51db4212019-06-12 06:59:12 -0700969 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
970 rtc::Event* destruction_event) const {
971 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
972 codec_height_);
973 }
974
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100975 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200976 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +0200977 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100978
979 video_source_.IncomingCapturedFrame(
980 CreateFrame(1, codec_width_, codec_height_));
981 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200982 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100983 }
984
sprang4847ae62017-06-27 07:06:52 -0700985 void WaitForEncodedFrame(int64_t expected_ntp_time) {
986 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200987 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700988 }
989
990 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
991 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200992 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700993 return ok;
994 }
995
996 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
997 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200998 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -0700999 }
1000
1001 void ExpectDroppedFrame() {
1002 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001003 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001004 }
1005
1006 bool WaitForFrame(int64_t timeout_ms) {
1007 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001008 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001009 return ok;
1010 }
1011
perkj26091b12016-09-01 01:17:40 -07001012 class TestEncoder : public test::FakeEncoder {
1013 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001014 explicit TestEncoder(TimeController* time_controller)
1015 : FakeEncoder(time_controller->GetClock()),
1016 time_controller_(time_controller) {
1017 RTC_DCHECK(time_controller_);
1018 }
perkj26091b12016-09-01 01:17:40 -07001019
Erik Språngaed30702018-11-05 12:57:17 +01001020 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +02001021 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +02001022 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +01001023 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001024 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +01001025 info.scaling_settings = VideoEncoder::ScalingSettings(
1026 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001027 }
1028 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001029 for (int i = 0; i < kMaxSpatialLayers; ++i) {
1030 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001031 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001032 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001033 for (int tid = 0; tid < num_layers; ++tid)
1034 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001035 }
1036 }
Erik Språngaed30702018-11-05 12:57:17 +01001037 }
Sergey Silkin6456e352019-07-08 17:56:40 +02001038
1039 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001040 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001041 info.apply_alignment_to_all_simulcast_layers =
1042 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001043 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001044 if (is_qp_trusted_.has_value()) {
1045 info.is_qp_trusted = is_qp_trusted_;
1046 }
Erik Språngaed30702018-11-05 12:57:17 +01001047 return info;
kthelgason876222f2016-11-29 01:44:11 -08001048 }
1049
Erik Språngb7cb7b52019-02-26 15:52:33 +01001050 int32_t RegisterEncodeCompleteCallback(
1051 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +02001052 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001053 encoded_image_callback_ = callback;
1054 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
1055 }
1056
perkjfa10b552016-10-02 23:45:26 -07001057 void ContinueEncode() { continue_encode_event_.Set(); }
1058
1059 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
1060 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +02001061 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -07001062 EXPECT_EQ(timestamp_, timestamp);
1063 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
1064 }
1065
kthelgason2fc52542017-03-03 00:24:41 -08001066 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +02001067 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -08001068 quality_scaling_ = b;
1069 }
kthelgasonad9010c2017-02-14 00:46:51 -08001070
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001071 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +02001072 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001073 requested_resolution_alignment_ = requested_resolution_alignment;
1074 }
1075
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001076 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
1077 MutexLock lock(&local_mutex_);
1078 apply_alignment_to_all_simulcast_layers_ = b;
1079 }
1080
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001081 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +02001082 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001083 is_hardware_accelerated_ = is_hardware_accelerated;
1084 }
1085
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001086 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
1087 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +02001088 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001089 temporal_layers_supported_[spatial_idx] = supported;
1090 }
1091
Sergey Silkin6456e352019-07-08 17:56:40 +02001092 void SetResolutionBitrateLimits(
1093 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +02001094 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +02001095 resolution_bitrate_limits_ = thresholds;
1096 }
1097
sprangfe627f32017-03-29 08:24:59 -07001098 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +02001099 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -07001100 force_init_encode_failed_ = force_failure;
1101 }
1102
Niels Möller6bb5ab92019-01-11 11:11:10 +01001103 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +02001104 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001105 rate_factor_ = rate_factor;
1106 }
1107
Erik Språngd7329ca2019-02-21 21:19:53 +01001108 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +02001109 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001110 return last_framerate_;
1111 }
1112
Erik Språngd7329ca2019-02-21 21:19:53 +01001113 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +02001114 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001115 return last_update_rect_;
1116 }
1117
Niels Möller87e2d782019-03-07 10:18:23 +01001118 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +02001119 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001120 return last_frame_types_;
1121 }
1122
1123 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +01001124 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +01001125 keyframe ? VideoFrameType::kVideoFrameKey
1126 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01001127 {
Markus Handella3765182020-07-08 13:13:32 +02001128 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001129 last_frame_types_ = frame_type;
1130 }
Niels Möllerb859b322019-03-07 12:40:01 +01001131 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +01001132 }
1133
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001134 void InjectEncodedImage(const EncodedImage& image,
1135 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +02001136 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001137 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001138 }
1139
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001140 void SetEncodedImageData(
1141 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +02001142 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001143 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001144 }
1145
Erik Språngd7329ca2019-02-21 21:19:53 +01001146 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +02001147 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001148 expect_null_frame_ = true;
1149 }
1150
Erik Språng5056af02019-09-02 15:53:11 +02001151 absl::optional<VideoEncoder::RateControlParameters>
1152 GetAndResetLastRateControlSettings() {
1153 auto settings = last_rate_control_settings_;
1154 last_rate_control_settings_.reset();
1155 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +01001156 }
1157
Henrik Boström56db9ff2021-03-24 09:06:45 +01001158 int GetLastInputWidth() const {
1159 MutexLock lock(&local_mutex_);
1160 return last_input_width_;
1161 }
1162
1163 int GetLastInputHeight() const {
1164 MutexLock lock(&local_mutex_);
1165 return last_input_height_;
1166 }
1167
Evan Shrubsole895556e2020-10-05 09:15:13 +02001168 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1169 MutexLock lock(&local_mutex_);
1170 return last_input_pixel_format_;
1171 }
1172
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001173 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001174 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001175 return num_set_rates_;
1176 }
1177
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001178 void SetPreferredPixelFormats(
1179 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1180 pixel_formats) {
1181 MutexLock lock(&local_mutex_);
1182 preferred_pixel_formats_ = std::move(pixel_formats);
1183 }
1184
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001185 void SetIsQpTrusted(absl::optional<bool> trusted) {
1186 MutexLock lock(&local_mutex_);
1187 is_qp_trusted_ = trusted;
1188 }
1189
perkjfa10b552016-10-02 23:45:26 -07001190 private:
perkj26091b12016-09-01 01:17:40 -07001191 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001192 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001193 {
Markus Handella3765182020-07-08 13:13:32 +02001194 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001195 if (expect_null_frame_) {
1196 EXPECT_EQ(input_image.timestamp(), 0u);
1197 EXPECT_EQ(input_image.width(), 1);
1198 last_frame_types_ = *frame_types;
1199 expect_null_frame_ = false;
1200 } else {
1201 EXPECT_GT(input_image.timestamp(), timestamp_);
1202 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1203 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1204 }
perkj26091b12016-09-01 01:17:40 -07001205
1206 timestamp_ = input_image.timestamp();
1207 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001208 last_input_width_ = input_image.width();
1209 last_input_height_ = input_image.height();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001210 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001211 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001212 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001213 }
Niels Möllerb859b322019-03-07 12:40:01 +01001214 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001215 return result;
1216 }
1217
Niels Möller08ae7ce2020-09-23 15:58:12 +02001218 CodecSpecificInfo EncodeHook(
1219 EncodedImage& encoded_image,
1220 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001221 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001222 {
1223 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001224 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001225 }
1226 MutexLock lock(&local_mutex_);
1227 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001228 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001229 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001230 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001231 }
1232
sprangfe627f32017-03-29 08:24:59 -07001233 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001234 const Settings& settings) override {
1235 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001236
Markus Handella3765182020-07-08 13:13:32 +02001237 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001238 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001239
Erik Språng82fad3d2018-03-21 09:57:23 +01001240 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001241 // Simulate setting up temporal layers, in order to validate the life
1242 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001243 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001244 frame_buffer_controller_ =
1245 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001246 }
Erik Språngb7cb7b52019-02-26 15:52:33 +01001247 if (force_init_encode_failed_) {
1248 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001249 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001250 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001251
Erik Språngb7cb7b52019-02-26 15:52:33 +01001252 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001253 return res;
1254 }
1255
Erik Språngb7cb7b52019-02-26 15:52:33 +01001256 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001257 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001258 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1259 initialized_ = EncoderState::kUninitialized;
1260 return FakeEncoder::Release();
1261 }
1262
Erik Språng16cb8f52019-04-12 13:59:09 +02001263 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001264 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001265 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001266 VideoBitrateAllocation adjusted_rate_allocation;
1267 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1268 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001269 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001270 adjusted_rate_allocation.SetBitrate(
1271 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001272 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001273 rate_factor_));
1274 }
1275 }
1276 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001277 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001278 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001279 RateControlParameters adjusted_paramters = parameters;
1280 adjusted_paramters.bitrate = adjusted_rate_allocation;
1281 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001282 }
1283
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001284 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001285 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001286 enum class EncoderState {
1287 kUninitialized,
1288 kInitializationFailed,
1289 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001290 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
perkj26091b12016-09-01 01:17:40 -07001291 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001292 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1293 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1294 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1295 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1296 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1297 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001298 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1299 false;
Markus Handella3765182020-07-08 13:13:32 +02001300 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001301 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1302 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001303 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001304 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001305 absl::optional<bool>
1306 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001307 local_mutex_);
1308 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1309 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1310 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001311 absl::optional<VideoEncoder::RateControlParameters>
1312 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001313 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1314 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001315 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001316 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001317 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1318 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001319 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 17:56:40 +02001320 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001321 RTC_GUARDED_BY(local_mutex_);
1322 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001323 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1324 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001325 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1326 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001327 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
perkj26091b12016-09-01 01:17:40 -07001328 };
1329
mflodmancc3d4422017-08-03 08:27:51 -07001330 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001331 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001332 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1333 : time_controller_(time_controller), test_encoder_(test_encoder) {
1334 RTC_DCHECK(time_controller_);
1335 }
perkj26091b12016-09-01 01:17:40 -07001336
perkj26091b12016-09-01 01:17:40 -07001337 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001338 EXPECT_TRUE(
1339 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1340 }
1341
1342 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1343 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001344 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001345 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001346 return false;
perkj26091b12016-09-01 01:17:40 -07001347 {
Markus Handella3765182020-07-08 13:13:32 +02001348 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001349 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001350 }
1351 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001352 return true;
perkj26091b12016-09-01 01:17:40 -07001353 }
1354
sprangb1ca0732017-02-01 08:38:12 -08001355 void WaitForEncodedFrame(uint32_t expected_width,
1356 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001357 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001358 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001359 }
1360
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001361 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001362 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001363 uint32_t width = 0;
1364 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001365 {
Markus Handella3765182020-07-08 13:13:32 +02001366 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001367 width = last_width_;
1368 height = last_height_;
1369 }
1370 EXPECT_EQ(expected_height, height);
1371 EXPECT_EQ(expected_width, width);
1372 }
1373
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001374 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1375 VideoRotation rotation;
1376 {
Markus Handella3765182020-07-08 13:13:32 +02001377 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001378 rotation = last_rotation_;
1379 }
1380 EXPECT_EQ(expected_rotation, rotation);
1381 }
1382
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001383 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001384
sprangc5d62e22017-04-02 23:53:04 -07001385 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001386 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +01001387 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001388 bool ret = encoded_frame_event_.Wait(timeout_ms);
Markus Handell28c71802021-11-08 10:11:55 +01001389 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001390 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001391 }
1392
perkj26091b12016-09-01 01:17:40 -07001393 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001394 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001395 expect_frames_ = false;
1396 }
1397
asaperssonfab67072017-04-04 05:51:49 -07001398 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001399 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001400 return number_of_reconfigurations_;
1401 }
1402
asaperssonfab67072017-04-04 05:51:49 -07001403 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001404 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001405 return min_transmit_bitrate_bps_;
1406 }
1407
Erik Språngd7329ca2019-02-21 21:19:53 +01001408 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001409 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001410 num_expected_layers_ = num_layers;
1411 }
1412
Erik Språngb7cb7b52019-02-26 15:52:33 +01001413 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001414 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001415 return last_capture_time_ms_;
1416 }
1417
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001418 const EncodedImage& GetLastEncodedImage() {
1419 MutexLock lock(&mutex_);
1420 return last_encoded_image_;
1421 }
1422
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001423 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001424 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001425 return std::move(last_encoded_image_data_);
1426 }
1427
Per Kjellanderdcef6412020-10-07 15:09:05 +02001428 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1429 MutexLock lock(&mutex_);
1430 return last_bitrate_allocation_;
1431 }
1432
1433 int number_of_bitrate_allocations() const {
1434 MutexLock lock(&mutex_);
1435 return number_of_bitrate_allocations_;
1436 }
1437
Per Kjellandera9434842020-10-15 17:53:22 +02001438 VideoLayersAllocation GetLastVideoLayersAllocation() {
1439 MutexLock lock(&mutex_);
1440 return last_layers_allocation_;
1441 }
1442
1443 int number_of_layers_allocations() const {
1444 MutexLock lock(&mutex_);
1445 return number_of_layers_allocations_;
1446 }
1447
perkj26091b12016-09-01 01:17:40 -07001448 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001449 Result OnEncodedImage(
1450 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001451 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001452 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001453 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001454 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001455 last_encoded_image_data_ = std::vector<uint8_t>(
1456 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001457 uint32_t timestamp = encoded_image.Timestamp();
1458 if (last_timestamp_ != timestamp) {
1459 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001460 last_width_ = encoded_image._encodedWidth;
1461 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001462 } else {
1463 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001464 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1465 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001466 }
1467 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001468 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001469 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001470 if (num_received_layers_ == num_expected_layers_) {
1471 encoded_frame_event_.Set();
1472 }
sprangb1ca0732017-02-01 08:38:12 -08001473 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001474 }
1475
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001476 void OnEncoderConfigurationChanged(
1477 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001478 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001479 VideoEncoderConfig::ContentType content_type,
1480 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001481 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001482 ++number_of_reconfigurations_;
1483 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1484 }
1485
Per Kjellanderdcef6412020-10-07 15:09:05 +02001486 void OnBitrateAllocationUpdated(
1487 const VideoBitrateAllocation& allocation) override {
1488 MutexLock lock(&mutex_);
1489 ++number_of_bitrate_allocations_;
1490 last_bitrate_allocation_ = allocation;
1491 }
1492
Per Kjellandera9434842020-10-15 17:53:22 +02001493 void OnVideoLayersAllocationUpdated(
1494 VideoLayersAllocation allocation) override {
1495 MutexLock lock(&mutex_);
1496 ++number_of_layers_allocations_;
1497 last_layers_allocation_ = allocation;
1498 rtc::StringBuilder log;
1499 for (const auto& layer : allocation.active_spatial_layers) {
1500 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1501 << "[";
1502 for (const auto target_bitrate :
1503 layer.target_bitrate_per_temporal_layer) {
1504 log << target_bitrate.kbps() << ",";
1505 }
1506 log << "]";
1507 }
Harald Alvestrand97597c02021-11-04 12:01:23 +00001508 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 17:53:22 +02001509 }
1510
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001511 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001512 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001513 TestEncoder* test_encoder_;
1514 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001515 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001516 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001517 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001518 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001519 uint32_t last_height_ = 0;
1520 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001521 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001522 size_t num_expected_layers_ = 1;
1523 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001524 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001525 int number_of_reconfigurations_ = 0;
1526 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001527 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1528 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001529 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1530 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001531 };
1532
Sergey Silkin5ee69672019-07-02 14:18:34 +02001533 class VideoBitrateAllocatorProxyFactory
1534 : public VideoBitrateAllocatorFactory {
1535 public:
1536 VideoBitrateAllocatorProxyFactory()
1537 : bitrate_allocator_factory_(
1538 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1539
1540 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1541 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001542 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001543 codec_config_ = codec;
1544 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1545 }
1546
1547 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001548 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001549 return codec_config_;
1550 }
1551
1552 private:
1553 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1554
Markus Handella3765182020-07-08 13:13:32 +02001555 mutable Mutex mutex_;
1556 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001557 };
1558
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001559 Clock* clock() { return time_controller_.GetClock(); }
1560 void AdvanceTime(TimeDelta duration) {
1561 time_controller_.AdvanceTime(duration);
1562 }
1563
1564 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1565
1566 protected:
1567 virtual TaskQueueFactory* GetTaskQueueFactory() {
1568 return time_controller_.GetTaskQueueFactory();
1569 }
1570
1571 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001572 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001573 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001574 int codec_width_;
1575 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001576 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001577 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001578 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001579 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001580 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001581 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001582 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001583 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001584};
1585
mflodmancc3d4422017-08-03 08:27:51 -07001586TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001587 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001588 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001589 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001590 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001591 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001592 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001593 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001594}
1595
mflodmancc3d4422017-08-03 08:27:51 -07001596TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001597 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001598 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001599 // The encoder will cache up to one frame for a short duration. Adding two
1600 // frames means that the first frame will be dropped and the second frame will
1601 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001602 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001603 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001604 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Markus Handell28c71802021-11-08 10:11:55 +01001605 AdvanceTime(TimeDelta::Zero());
perkja49cbd32016-09-16 07:53:41 -07001606 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001607
Henrik Boström381d1092020-05-12 18:49:07 +02001608 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001609 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001610
Sebastian Janssona3177052018-04-10 13:05:49 +02001611 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001612 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001613 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1614
1615 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001616 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001617}
1618
mflodmancc3d4422017-08-03 08:27:51 -07001619TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001620 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001621 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001622 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001623 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001624
Henrik Boström381d1092020-05-12 18:49:07 +02001625 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001626 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1627
Sebastian Janssona3177052018-04-10 13:05:49 +02001628 // The encoder will cache up to one frame for a short duration. Adding two
1629 // frames means that the first frame will be dropped and the second frame will
1630 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001631 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001632 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001633
Henrik Boström381d1092020-05-12 18:49:07 +02001634 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001635 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001636 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001637 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1638 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001639 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001640}
1641
mflodmancc3d4422017-08-03 08:27:51 -07001642TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001643 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001644 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001645 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001646 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001647
1648 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001649 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001650
perkja49cbd32016-09-16 07:53:41 -07001651 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001652 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001653 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001654}
1655
mflodmancc3d4422017-08-03 08:27:51 -07001656TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001657 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001658 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001659
perkja49cbd32016-09-16 07:53:41 -07001660 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001661 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001662
mflodmancc3d4422017-08-03 08:27:51 -07001663 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001664 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001665 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001666 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1667 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001668}
1669
Markus Handell9a478b52021-11-18 16:07:01 +01001670TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
1671 test::FrameForwarder source;
1672 video_stream_encoder_->SetSource(&source,
1673 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02001674 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001675 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001676
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001677 int dropped_count = 0;
1678 stats_proxy_->SetDroppedFrameCallback(
1679 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1680 ++dropped_count;
1681 });
1682
Markus Handell9a478b52021-11-18 16:07:01 +01001683 source.IncomingCapturedFrame(CreateFrame(1, nullptr));
1684 source.IncomingCapturedFrame(CreateFrame(2, nullptr));
1685 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001686 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001687 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001688}
1689
Henrik Boström56db9ff2021-03-24 09:06:45 +01001690TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001691 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001692 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001693
1694 rtc::Event frame_destroyed_event;
1695 video_source_.IncomingCapturedFrame(
1696 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001697 WaitForEncodedFrame(1);
1698 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1699 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001700 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1701 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001702 video_stream_encoder_->Stop();
1703}
1704
Henrik Boström56db9ff2021-03-24 09:06:45 +01001705TEST_F(VideoStreamEncoderTest,
1706 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001707 // Use the cropping factory.
1708 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001709 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001710 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1711 kMaxPayloadLength);
1712 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1713
1714 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001715 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001716 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001717 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1718 WaitForEncodedFrame(1);
1719 // The encoder will have been configured once.
1720 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001721 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1722 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 06:59:12 -07001723
1724 // Now send in a fake frame that needs to be cropped as the width/height
1725 // aren't divisible by 4 (see CreateEncoderStreams above).
1726 rtc::Event frame_destroyed_event;
1727 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1728 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001729 WaitForEncodedFrame(2);
1730 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1731 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001732 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1733 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001734 video_stream_encoder_->Stop();
1735}
1736
Evan Shrubsole895556e2020-10-05 09:15:13 +02001737TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1738 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001739 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001740
1741 video_source_.IncomingCapturedFrame(
1742 CreateNV12Frame(1, codec_width_, codec_height_));
1743 WaitForEncodedFrame(1);
1744 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1745 fake_encoder_.GetLastInputPixelFormat());
1746 video_stream_encoder_->Stop();
1747}
1748
Henrik Boström56db9ff2021-03-24 09:06:45 +01001749TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001750 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001751 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001752
1753 fake_encoder_.SetPreferredPixelFormats({});
1754
1755 rtc::Event frame_destroyed_event;
1756 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1757 1, &frame_destroyed_event, codec_width_, codec_height_));
1758 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001759 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001760 fake_encoder_.GetLastInputPixelFormat());
1761 video_stream_encoder_->Stop();
1762}
1763
Henrik Boström56db9ff2021-03-24 09:06:45 +01001764TEST_F(VideoStreamEncoderTest,
1765 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001766 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001767 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001768
1769 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1770
1771 rtc::Event frame_destroyed_event;
1772 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1773 1, &frame_destroyed_event, codec_width_, codec_height_));
1774 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001775 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001776 fake_encoder_.GetLastInputPixelFormat());
1777 video_stream_encoder_->Stop();
1778}
1779
Henrik Boström56db9ff2021-03-24 09:06:45 +01001780TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001781 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001782 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001783
1784 // Fake NV12 native frame does not allow mapping to I444.
1785 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1786
1787 rtc::Event frame_destroyed_event;
1788 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1789 1, &frame_destroyed_event, codec_width_, codec_height_));
1790 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001791 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001792 fake_encoder_.GetLastInputPixelFormat());
1793 video_stream_encoder_->Stop();
1794}
1795
Henrik Boström56db9ff2021-03-24 09:06:45 +01001796TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001797 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001798 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001799
1800 rtc::Event frame_destroyed_event;
1801 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1802 1, &frame_destroyed_event, codec_width_, codec_height_));
1803 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001804 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001805 fake_encoder_.GetLastInputPixelFormat());
1806 video_stream_encoder_->Stop();
1807}
1808
Ying Wang9b881ab2020-02-07 14:29:32 +01001809TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001810 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001811 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001812 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1813 WaitForEncodedFrame(1);
1814
Henrik Boström381d1092020-05-12 18:49:07 +02001815 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001816 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001817 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1818 // frames. Adding two frames means that the first frame will be dropped and
1819 // the second frame will be sent to the encoder.
1820 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1821 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1822 WaitForEncodedFrame(3);
1823 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1824 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1825 WaitForEncodedFrame(5);
1826 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1827 video_stream_encoder_->Stop();
1828}
1829
mflodmancc3d4422017-08-03 08:27:51 -07001830TEST_F(VideoStreamEncoderTest,
1831 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001832 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001833 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001834 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001835
1836 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001837 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001838 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001839 // The encoder will have been configured once when the first frame is
1840 // received.
1841 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001842
1843 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001844 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001845 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001846 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001847 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001848
1849 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001850 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001851 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001852 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001853 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001854
mflodmancc3d4422017-08-03 08:27:51 -07001855 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001856}
1857
mflodmancc3d4422017-08-03 08:27:51 -07001858TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001859 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001860 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001861
1862 // Capture a frame and wait for it to synchronize with the encoder thread.
1863 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001864 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001865 // The encoder will have been configured once.
1866 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001867 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1868 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-02 23:45:26 -07001869
1870 codec_width_ *= 2;
1871 codec_height_ *= 2;
1872 // Capture a frame with a higher resolution and wait for it to synchronize
1873 // with the encoder thread.
1874 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001875 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 10:07:11 +02001876 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1877 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 21:37:57 +01001878 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001879
mflodmancc3d4422017-08-03 08:27:51 -07001880 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001881}
1882
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001883TEST_F(VideoStreamEncoderTest,
1884 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001885 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001886 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001887
1888 // Capture a frame and wait for it to synchronize with the encoder thread.
1889 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1890 WaitForEncodedFrame(1);
1891
1892 VideoEncoderConfig video_encoder_config;
1893 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1894 // Changing the max payload data length recreates encoder.
1895 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1896 kMaxPayloadLength / 2);
1897
1898 // Capture a frame and wait for it to synchronize with the encoder thread.
1899 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1900 WaitForEncodedFrame(2);
1901 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1902
1903 video_stream_encoder_->Stop();
1904}
1905
Sergey Silkin5ee69672019-07-02 14:18:34 +02001906TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001907 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001908 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001909
1910 VideoEncoderConfig video_encoder_config;
1911 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02001912 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1913 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001914 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1915 kMaxPayloadLength);
1916
1917 // Capture a frame and wait for it to synchronize with the encoder thread.
1918 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1919 WaitForEncodedFrame(1);
1920 // The encoder will have been configured once when the first frame is
1921 // received.
1922 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001923 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001924 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001925 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001926 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1927
Sergey Silkin6456e352019-07-08 17:56:40 +02001928 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1929 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 10:07:11 +02001930 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1931 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001932 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1933 kMaxPayloadLength);
1934
1935 // Capture a frame and wait for it to synchronize with the encoder thread.
1936 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1937 WaitForEncodedFrame(2);
1938 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1939 // Bitrate limits have changed - rate allocator should be reconfigured,
1940 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 10:07:11 +02001941 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001942 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001943 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001944 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001945 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001946
1947 video_stream_encoder_->Stop();
1948}
1949
Sergey Silkin6456e352019-07-08 17:56:40 +02001950TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001951 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001952 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001953 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001954
Sergey Silkincd02eba2020-01-20 14:48:40 +01001955 const uint32_t kMinEncBitrateKbps = 100;
1956 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001957 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001958 /*frame_size_pixels=*/codec_width_ * codec_height_,
1959 /*min_start_bitrate_bps=*/0,
1960 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1961 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001962 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1963
Sergey Silkincd02eba2020-01-20 14:48:40 +01001964 VideoEncoderConfig video_encoder_config;
1965 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1966 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1967 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1968 (kMinEncBitrateKbps + 1) * 1000;
1969 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1970 kMaxPayloadLength);
1971
1972 // When both encoder and app provide bitrate limits, the intersection of
1973 // provided sets should be used.
1974 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1975 WaitForEncodedFrame(1);
1976 EXPECT_EQ(kMaxEncBitrateKbps,
1977 bitrate_allocator_factory_.codec_config().maxBitrate);
1978 EXPECT_EQ(kMinEncBitrateKbps + 1,
1979 bitrate_allocator_factory_.codec_config().minBitrate);
1980
1981 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
1982 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1983 (kMinEncBitrateKbps - 1) * 1000;
1984 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1985 kMaxPayloadLength);
1986 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001987 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001988 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001989 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01001990 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001991 bitrate_allocator_factory_.codec_config().minBitrate);
1992
Sergey Silkincd02eba2020-01-20 14:48:40 +01001993 video_stream_encoder_->Stop();
1994}
1995
1996TEST_F(VideoStreamEncoderTest,
1997 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02001998 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001999 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002000
2001 const uint32_t kMinAppBitrateKbps = 100;
2002 const uint32_t kMaxAppBitrateKbps = 200;
2003 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
2004 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
2005 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2006 /*frame_size_pixels=*/codec_width_ * codec_height_,
2007 /*min_start_bitrate_bps=*/0,
2008 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
2009 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
2010 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2011
2012 VideoEncoderConfig video_encoder_config;
2013 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2014 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
2015 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2016 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002017 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2018 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002019
Sergey Silkincd02eba2020-01-20 14:48:40 +01002020 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2021 WaitForEncodedFrame(1);
2022 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002023 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002024 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002025 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02002026
2027 video_stream_encoder_->Stop();
2028}
2029
2030TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002031 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02002032 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002033 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02002034
2035 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002036 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002037 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002038 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002039 fake_encoder_.SetResolutionBitrateLimits(
2040 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
2041
2042 VideoEncoderConfig video_encoder_config;
2043 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2044 video_encoder_config.max_bitrate_bps = 0;
2045 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2046 kMaxPayloadLength);
2047
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002048 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002049 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2050 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002051 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2052 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002053 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2054 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2055
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002056 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002057 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2058 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002059 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2060 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002061 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2062 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2063
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002064 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02002065 // encoder for 360p should be used.
2066 video_source_.IncomingCapturedFrame(
2067 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2068 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002069 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2070 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002071 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2072 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2073
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002074 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02002075 // ignored.
2076 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2077 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002078 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2079 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002080 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2081 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002082 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2083 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002084 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2085 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2086
2087 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2088 // for 270p should be used.
2089 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2090 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002091 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2092 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002093 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2094 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2095
2096 video_stream_encoder_->Stop();
2097}
2098
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002099TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002100 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002101 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002102
2103 VideoEncoderConfig video_encoder_config;
2104 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2105 video_encoder_config.max_bitrate_bps = 0;
2106 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2107 kMaxPayloadLength);
2108
2109 // Encode 720p frame to get the default encoder target bitrate.
2110 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2111 WaitForEncodedFrame(1);
2112 const uint32_t kDefaultTargetBitrateFor720pKbps =
2113 bitrate_allocator_factory_.codec_config()
2114 .simulcastStream[0]
2115 .targetBitrate;
2116
2117 // Set the max recommended encoder bitrate to something lower than the default
2118 // target bitrate.
2119 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2120 1280 * 720, 10 * 1000, 10 * 1000,
2121 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2122 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2123
2124 // Change resolution to trigger encoder reinitialization.
2125 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2126 WaitForEncodedFrame(2);
2127 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2128 WaitForEncodedFrame(3);
2129
2130 // Ensure the target bitrate is capped by the max bitrate.
2131 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2132 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2133 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2134 .simulcastStream[0]
2135 .targetBitrate *
2136 1000,
2137 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2138
2139 video_stream_encoder_->Stop();
2140}
2141
Åsa Perssona7e34d32021-01-20 15:36:13 +01002142TEST_F(VideoStreamEncoderTest,
2143 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2144 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2145 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2146 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2147 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2148 fake_encoder_.SetResolutionBitrateLimits(
2149 {kEncoderLimits270p, kEncoderLimits360p});
2150
2151 // Two streams, highest stream active.
2152 VideoEncoderConfig config;
2153 const int kNumStreams = 2;
2154 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2155 config.max_bitrate_bps = 0;
2156 config.simulcast_layers[0].active = false;
2157 config.simulcast_layers[1].active = true;
2158 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002159 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002160 "VP8", /*max qp*/ 56, /*screencast*/ false,
2161 /*screenshare enabled*/ false);
2162 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2163
2164 // The encoder bitrate limits for 270p should be used.
2165 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2166 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002167 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002168 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002169 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002170 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002171 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002172
2173 // The encoder bitrate limits for 360p should be used.
2174 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2175 EXPECT_FALSE(WaitForFrame(1000));
2176 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002177 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002178 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002179 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002180
2181 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2182 video_source_.IncomingCapturedFrame(
2183 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2184 EXPECT_FALSE(WaitForFrame(1000));
2185 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002186 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002187 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002188 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002189
2190 // Resolution higher than 360p. Encoder limits should be ignored.
2191 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2192 EXPECT_FALSE(WaitForFrame(1000));
2193 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002194 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002195 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002196 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002197 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002198 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002199 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002200 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002201
2202 // Resolution lower than 270p. The encoder limits for 270p should be used.
2203 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2204 EXPECT_FALSE(WaitForFrame(1000));
2205 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002206 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002207 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002208 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002209
2210 video_stream_encoder_->Stop();
2211}
2212
2213TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002214 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2215 // Two streams, highest stream active.
2216 VideoEncoderConfig config;
2217 const int kNumStreams = 2;
2218 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2219 config.max_bitrate_bps = 0;
2220 config.simulcast_layers[0].active = false;
2221 config.simulcast_layers[1].active = true;
2222 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002223 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002224 "VP8", /*max qp*/ 56, /*screencast*/ false,
2225 /*screenshare enabled*/ false);
2226 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2227
2228 // Default bitrate limits for 270p should be used.
2229 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2230 kDefaultLimits270p =
2231 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002232 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002233 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2234 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002235 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 10:39:51 +01002236 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002237 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002238 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002239 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002240
2241 // Default bitrate limits for 360p should be used.
2242 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2243 kDefaultLimits360p =
2244 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002245 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002246 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2247 EXPECT_FALSE(WaitForFrame(1000));
2248 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002249 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002250 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002251 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002252
2253 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2254 video_source_.IncomingCapturedFrame(
2255 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2256 EXPECT_FALSE(WaitForFrame(1000));
2257 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002258 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002259 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002260 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002261
2262 // Default bitrate limits for 540p should be used.
2263 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2264 kDefaultLimits540p =
2265 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002266 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002267 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2268 EXPECT_FALSE(WaitForFrame(1000));
2269 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002270 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002271 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002272 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002273
2274 video_stream_encoder_->Stop();
2275}
2276
2277TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002278 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2279 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2280 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2281 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2282 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2283 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2284 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2285 fake_encoder_.SetResolutionBitrateLimits(
2286 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2287
2288 // Three streams, middle stream active.
2289 VideoEncoderConfig config;
2290 const int kNumStreams = 3;
2291 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2292 config.simulcast_layers[0].active = false;
2293 config.simulcast_layers[1].active = true;
2294 config.simulcast_layers[2].active = false;
2295 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002296 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002297 "VP8", /*max qp*/ 56, /*screencast*/ false,
2298 /*screenshare enabled*/ false);
2299 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2300
2301 // The encoder bitrate limits for 360p should be used.
2302 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2303 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002304 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002305 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002306 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002307 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002308 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002309
2310 // The encoder bitrate limits for 270p should be used.
2311 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2312 EXPECT_FALSE(WaitForFrame(1000));
2313 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002314 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002315 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002316 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002317
2318 video_stream_encoder_->Stop();
2319}
2320
2321TEST_F(VideoStreamEncoderTest,
2322 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2323 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2324 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2325 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2326 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2327 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2328 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2329 fake_encoder_.SetResolutionBitrateLimits(
2330 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2331
2332 // Three streams, lowest stream active.
2333 VideoEncoderConfig config;
2334 const int kNumStreams = 3;
2335 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2336 config.simulcast_layers[0].active = true;
2337 config.simulcast_layers[1].active = false;
2338 config.simulcast_layers[2].active = false;
2339 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002340 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002341 "VP8", /*max qp*/ 56, /*screencast*/ false,
2342 /*screenshare enabled*/ false);
2343 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2344
2345 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2346 // on lowest stream, limits for 270p should not be used
2347 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2348 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002349 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002350 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002351 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002352 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002353 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002354
2355 video_stream_encoder_->Stop();
2356}
2357
2358TEST_F(VideoStreamEncoderTest,
2359 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2360 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2361 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2362 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2363 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2364 fake_encoder_.SetResolutionBitrateLimits(
2365 {kEncoderLimits270p, kEncoderLimits360p});
2366 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2367
2368 // Two streams, highest stream active.
2369 VideoEncoderConfig config;
2370 const int kNumStreams = 2;
2371 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2372 config.simulcast_layers[0].active = false;
2373 config.simulcast_layers[1].active = true;
2374 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2375 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002376 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002377 "VP8", /*max qp*/ 56, /*screencast*/ false,
2378 /*screenshare enabled*/ false);
2379 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2380
2381 // The encoder bitrate limits for 270p should be used.
2382 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2383 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002384 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002385 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002386 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002387 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002388 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002389
2390 // The max configured bitrate is less than the encoder limit for 360p.
2391 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2392 EXPECT_FALSE(WaitForFrame(1000));
2393 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002394 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002395 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002396 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002397
2398 video_stream_encoder_->Stop();
2399}
2400
mflodmancc3d4422017-08-03 08:27:51 -07002401TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002402 EXPECT_TRUE(video_source_.has_sinks());
2403 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002404 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002405 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002406 EXPECT_FALSE(video_source_.has_sinks());
2407 EXPECT_TRUE(new_video_source.has_sinks());
2408
mflodmancc3d4422017-08-03 08:27:51 -07002409 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002410}
2411
mflodmancc3d4422017-08-03 08:27:51 -07002412TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002413 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002414 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002415 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002416 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002417}
2418
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002419class ResolutionAlignmentTest
2420 : public VideoStreamEncoderTest,
2421 public ::testing::WithParamInterface<
2422 ::testing::tuple<int, std::vector<double>>> {
2423 public:
2424 ResolutionAlignmentTest()
2425 : requested_alignment_(::testing::get<0>(GetParam())),
2426 scale_factors_(::testing::get<1>(GetParam())) {}
2427
2428 protected:
2429 const int requested_alignment_;
2430 const std::vector<double> scale_factors_;
2431};
2432
2433INSTANTIATE_TEST_SUITE_P(
2434 AlignmentAndScaleFactors,
2435 ResolutionAlignmentTest,
2436 ::testing::Combine(
2437 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2438 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2439 std::vector<double>{-1.0, -1.0},
2440 std::vector<double>{-1.0, -1.0, -1.0},
2441 std::vector<double>{4.0, 2.0, 1.0},
2442 std::vector<double>{9999.0, -1.0, 1.0},
2443 std::vector<double>{3.99, 2.01, 1.0},
2444 std::vector<double>{4.9, 1.7, 1.25},
2445 std::vector<double>{10.0, 4.0, 3.0},
2446 std::vector<double>{1.75, 3.5},
2447 std::vector<double>{1.5, 2.5},
2448 std::vector<double>{1.3, 1.0})));
2449
2450TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2451 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002452 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002453 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2454 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2455
2456 // Fill config with the scaling factor by which to reduce encoding size.
2457 const int num_streams = scale_factors_.size();
2458 VideoEncoderConfig config;
2459 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2460 for (int i = 0; i < num_streams; ++i) {
2461 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2462 }
2463 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002464 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002465 "VP8", /*max qp*/ 56, /*screencast*/ false,
2466 /*screenshare enabled*/ false);
2467 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2468
Henrik Boström381d1092020-05-12 18:49:07 +02002469 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002470 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2471 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002472 // Wait for all layers before triggering event.
2473 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002474
2475 // On the 1st frame, we should have initialized the encoder and
2476 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002477 int64_t timestamp_ms = kFrameIntervalMs;
2478 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2479 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002480 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002481
2482 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2483 // (It's up the to the encoder to potentially drop the previous frame,
2484 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002485 timestamp_ms += kFrameIntervalMs;
2486 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2487 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002488 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002489
Asa Persson606d3cb2021-10-04 10:07:11 +02002490 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002491 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2492 // Frame size should be a multiple of the requested alignment.
2493 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2494 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2495 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2496 // Aspect ratio should match.
2497 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2498 codec.height * codec.simulcastStream[i].width);
2499 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002500
2501 video_stream_encoder_->Stop();
2502}
2503
Jonathan Yubc771b72017-12-08 17:04:29 -08002504TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2505 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002506 const int kWidth = 1280;
2507 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002508
2509 // We rely on the automatic resolution adaptation, but we handle framerate
2510 // adaptation manually by mocking the stats proxy.
2511 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002512
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002513 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002514 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002515 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002516 video_stream_encoder_->SetSource(&video_source_,
2517 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002518 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002519 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002520 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002521 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2522
Jonathan Yubc771b72017-12-08 17:04:29 -08002523 // Adapt down as far as possible.
2524 rtc::VideoSinkWants last_wants;
2525 int64_t t = 1;
2526 int loop_count = 0;
2527 do {
2528 ++loop_count;
2529 last_wants = video_source_.sink_wants();
2530
2531 // Simulate the framerate we've been asked to adapt to.
2532 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2533 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2534 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2535 mock_stats.input_frame_rate = fps;
2536 stats_proxy_->SetMockStats(mock_stats);
2537
2538 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2539 sink_.WaitForEncodedFrame(t);
2540 t += frame_interval_ms;
2541
mflodmancc3d4422017-08-03 08:27:51 -07002542 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002543 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002544 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002545 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2546 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002547 } while (video_source_.sink_wants().max_pixel_count <
2548 last_wants.max_pixel_count ||
2549 video_source_.sink_wants().max_framerate_fps <
2550 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002551
Jonathan Yubc771b72017-12-08 17:04:29 -08002552 // Verify that we've adapted all the way down.
2553 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002554 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002555 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2556 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002557 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002558 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2559 *video_source_.last_sent_height());
2560 EXPECT_EQ(kMinBalancedFramerateFps,
2561 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002562
Jonathan Yubc771b72017-12-08 17:04:29 -08002563 // Adapt back up the same number of times we adapted down.
2564 for (int i = 0; i < loop_count - 1; ++i) {
2565 last_wants = video_source_.sink_wants();
2566
2567 // Simulate the framerate we've been asked to adapt to.
2568 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2569 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2570 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2571 mock_stats.input_frame_rate = fps;
2572 stats_proxy_->SetMockStats(mock_stats);
2573
2574 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2575 sink_.WaitForEncodedFrame(t);
2576 t += frame_interval_ms;
2577
Henrik Boström91aa7322020-04-28 12:24:33 +02002578 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002579 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002580 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002581 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2582 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002583 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2584 last_wants.max_pixel_count ||
2585 video_source_.sink_wants().max_framerate_fps >
2586 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002587 }
2588
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002589 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002590 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002591 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002592 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2593 EXPECT_EQ((loop_count - 1) * 2,
2594 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002595
mflodmancc3d4422017-08-03 08:27:51 -07002596 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002597}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002598
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002599TEST_F(VideoStreamEncoderTest,
2600 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02002601 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2602 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002603 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002604
2605 const int kFrameWidth = 1280;
2606 const int kFrameHeight = 720;
2607
2608 int64_t ntp_time = kFrameIntervalMs;
2609
2610 // Force an input frame rate to be available, or the adaptation call won't
2611 // know what framerate to adapt form.
2612 const int kInputFps = 30;
2613 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2614 stats.input_frame_rate = kInputFps;
2615 stats_proxy_->SetMockStats(stats);
2616
2617 video_source_.set_adaptation_enabled(true);
2618 video_stream_encoder_->SetSource(
2619 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002620 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002621 video_source_.IncomingCapturedFrame(
2622 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2623 sink_.WaitForEncodedFrame(ntp_time);
2624 ntp_time += kFrameIntervalMs;
2625
2626 // Trigger CPU overuse.
2627 video_stream_encoder_->TriggerCpuOveruse();
2628 video_source_.IncomingCapturedFrame(
2629 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2630 sink_.WaitForEncodedFrame(ntp_time);
2631 ntp_time += kFrameIntervalMs;
2632
2633 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2634 EXPECT_EQ(std::numeric_limits<int>::max(),
2635 video_source_.sink_wants().max_pixel_count);
2636 // Some framerate constraint should be set.
2637 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2638 EXPECT_LT(restricted_fps, kInputFps);
2639 video_source_.IncomingCapturedFrame(
2640 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2641 sink_.WaitForEncodedFrame(ntp_time);
2642 ntp_time += 100;
2643
Henrik Boström2671dac2020-05-19 16:29:09 +02002644 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002645 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2646 // Give the encoder queue time to process the change in degradation preference
2647 // by waiting for an encoded frame.
2648 video_source_.IncomingCapturedFrame(
2649 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2650 sink_.WaitForEncodedFrame(ntp_time);
2651 ntp_time += kFrameIntervalMs;
2652
2653 video_stream_encoder_->TriggerQualityLow();
2654 video_source_.IncomingCapturedFrame(
2655 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2656 sink_.WaitForEncodedFrame(ntp_time);
2657 ntp_time += kFrameIntervalMs;
2658
2659 // Some resolution constraint should be set.
2660 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2661 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2662 kFrameWidth * kFrameHeight);
2663 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2664
2665 int pixel_count = video_source_.sink_wants().max_pixel_count;
2666 // Triggering a CPU underuse should not change the sink wants since it has
2667 // not been overused for resolution since we changed degradation preference.
2668 video_stream_encoder_->TriggerCpuUnderuse();
2669 video_source_.IncomingCapturedFrame(
2670 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2671 sink_.WaitForEncodedFrame(ntp_time);
2672 ntp_time += kFrameIntervalMs;
2673 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2674 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2675
Evan Shrubsole64469032020-06-11 10:45:29 +02002676 // Change the degradation preference back. CPU underuse should not adapt since
2677 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002678 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002679 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2680 video_source_.IncomingCapturedFrame(
2681 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2682 sink_.WaitForEncodedFrame(ntp_time);
2683 ntp_time += 100;
2684 // Resolution adaptations is gone after changing degradation preference.
2685 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2686 EXPECT_EQ(std::numeric_limits<int>::max(),
2687 video_source_.sink_wants().max_pixel_count);
2688 // The fps adaptation from above is now back.
2689 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2690
2691 // Trigger CPU underuse.
2692 video_stream_encoder_->TriggerCpuUnderuse();
2693 video_source_.IncomingCapturedFrame(
2694 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2695 sink_.WaitForEncodedFrame(ntp_time);
2696 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002697 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2698
2699 // Trigger QP underuse, fps should return to normal.
2700 video_stream_encoder_->TriggerQualityHigh();
2701 video_source_.IncomingCapturedFrame(
2702 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2703 sink_.WaitForEncodedFrame(ntp_time);
2704 ntp_time += kFrameIntervalMs;
2705 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002706
2707 video_stream_encoder_->Stop();
2708}
2709
mflodmancc3d4422017-08-03 08:27:51 -07002710TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002711 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002712 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002713 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002714
sprangc5d62e22017-04-02 23:53:04 -07002715 const int kFrameWidth = 1280;
2716 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002717
Åsa Persson8c1bf952018-09-13 10:42:19 +02002718 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002719
kthelgason5e13d412016-12-01 03:59:51 -08002720 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002721 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002722 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002723 frame_timestamp += kFrameIntervalMs;
2724
perkj803d97f2016-11-01 11:45:46 -07002725 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002726 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002727 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002728 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002729 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002730 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002731
asapersson0944a802017-04-07 00:57:58 -07002732 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002733 // wanted resolution.
2734 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2735 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2736 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002737 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002738
2739 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002740 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002741 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002742 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002743 // Give the encoder queue time to process the change in degradation preference
2744 // by waiting for an encoded frame.
2745 new_video_source.IncomingCapturedFrame(
2746 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2747 sink_.WaitForEncodedFrame(frame_timestamp);
2748 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002749 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002750 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002751
sprangc5d62e22017-04-02 23:53:04 -07002752 // Force an input frame rate to be available, or the adaptation call won't
2753 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002754 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002755 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002756 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002757 stats_proxy_->SetMockStats(stats);
2758
mflodmancc3d4422017-08-03 08:27:51 -07002759 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002760 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002761 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002762 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002763 frame_timestamp += kFrameIntervalMs;
2764
2765 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002766 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002767 EXPECT_EQ(std::numeric_limits<int>::max(),
2768 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002769 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002770
asapersson02465b82017-04-10 01:12:52 -07002771 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002772 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2773 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002774 // Give the encoder queue time to process the change in degradation preference
2775 // by waiting for an encoded frame.
2776 new_video_source.IncomingCapturedFrame(
2777 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2778 sink_.WaitForEncodedFrame(frame_timestamp);
2779 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002780 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002781
mflodmancc3d4422017-08-03 08:27:51 -07002782 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002783 new_video_source.IncomingCapturedFrame(
2784 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002785 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002786 frame_timestamp += kFrameIntervalMs;
2787
2788 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002789 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002790
2791 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002792 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002793 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002794 // Give the encoder queue time to process the change in degradation preference
2795 // by waiting for an encoded frame.
2796 new_video_source.IncomingCapturedFrame(
2797 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2798 sink_.WaitForEncodedFrame(frame_timestamp);
2799 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002800 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2801 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002802 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002803 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002804
2805 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002806 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002807 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002808 // Give the encoder queue time to process the change in degradation preference
2809 // by waiting for an encoded frame.
2810 new_video_source.IncomingCapturedFrame(
2811 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2812 sink_.WaitForEncodedFrame(frame_timestamp);
2813 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002814 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2815 EXPECT_EQ(std::numeric_limits<int>::max(),
2816 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002817 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002818
mflodmancc3d4422017-08-03 08:27:51 -07002819 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002820}
2821
mflodmancc3d4422017-08-03 08:27:51 -07002822TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002823 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002824 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002825
asaperssonfab67072017-04-04 05:51:49 -07002826 const int kWidth = 1280;
2827 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002828 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002829 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002830 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2831 EXPECT_FALSE(stats.bw_limited_resolution);
2832 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2833
2834 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002835 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002836 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002837 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002838
2839 stats = stats_proxy_->GetStats();
2840 EXPECT_TRUE(stats.bw_limited_resolution);
2841 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2842
2843 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002844 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002845 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002846 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002847
2848 stats = stats_proxy_->GetStats();
2849 EXPECT_FALSE(stats.bw_limited_resolution);
2850 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2851 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2852
mflodmancc3d4422017-08-03 08:27:51 -07002853 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002854}
2855
mflodmancc3d4422017-08-03 08:27:51 -07002856TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002857 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002858 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002859
2860 const int kWidth = 1280;
2861 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002862 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002863 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002864 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2865 EXPECT_FALSE(stats.cpu_limited_resolution);
2866 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2867
2868 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002869 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002870 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002871 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002872
2873 stats = stats_proxy_->GetStats();
2874 EXPECT_TRUE(stats.cpu_limited_resolution);
2875 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2876
2877 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002878 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002879 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002880 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002881
2882 stats = stats_proxy_->GetStats();
2883 EXPECT_FALSE(stats.cpu_limited_resolution);
2884 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002885 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002886
mflodmancc3d4422017-08-03 08:27:51 -07002887 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002888}
2889
mflodmancc3d4422017-08-03 08:27:51 -07002890TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002891 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002892 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002893
asaperssonfab67072017-04-04 05:51:49 -07002894 const int kWidth = 1280;
2895 const int kHeight = 720;
2896 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002897 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002898 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002899 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002900 EXPECT_FALSE(stats.cpu_limited_resolution);
2901 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2902
asaperssonfab67072017-04-04 05:51:49 -07002903 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002904 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002905 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002906 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002907 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002908 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002909 EXPECT_TRUE(stats.cpu_limited_resolution);
2910 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2911
2912 // Set new source with adaptation still enabled.
2913 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002914 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002915 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002916
asaperssonfab67072017-04-04 05:51:49 -07002917 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002918 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002919 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002920 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002921 EXPECT_TRUE(stats.cpu_limited_resolution);
2922 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2923
2924 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002925 video_stream_encoder_->SetSource(&new_video_source,
2926 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002927
asaperssonfab67072017-04-04 05:51:49 -07002928 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002929 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002930 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002931 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002932 EXPECT_FALSE(stats.cpu_limited_resolution);
2933 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2934
2935 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002936 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002937 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002938
asaperssonfab67072017-04-04 05:51:49 -07002939 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002940 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002941 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002942 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002943 EXPECT_TRUE(stats.cpu_limited_resolution);
2944 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2945
asaperssonfab67072017-04-04 05:51:49 -07002946 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002947 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002948 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002949 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002950 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002951 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002952 EXPECT_FALSE(stats.cpu_limited_resolution);
2953 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002954 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002955
mflodmancc3d4422017-08-03 08:27:51 -07002956 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002957}
2958
mflodmancc3d4422017-08-03 08:27:51 -07002959TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002960 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002961 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002962
asaperssonfab67072017-04-04 05:51:49 -07002963 const int kWidth = 1280;
2964 const int kHeight = 720;
2965 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002966 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002967 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002968 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002969 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002970 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002971
2972 // Set new source with adaptation still enabled.
2973 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002974 video_stream_encoder_->SetSource(&new_video_source,
2975 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002976
asaperssonfab67072017-04-04 05:51:49 -07002977 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002978 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002979 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002980 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002981 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002982 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002983
asaperssonfab67072017-04-04 05:51:49 -07002984 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002985 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002986 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002987 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002988 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002989 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002990 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002991 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002992
asaperssonfab67072017-04-04 05:51:49 -07002993 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002994 video_stream_encoder_->SetSource(&new_video_source,
2995 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08002996
asaperssonfab67072017-04-04 05:51:49 -07002997 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002998 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002999 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003000 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003001 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003002 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003003
asapersson02465b82017-04-10 01:12:52 -07003004 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07003005 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003006 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003007
asaperssonfab67072017-04-04 05:51:49 -07003008 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003009 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08003010 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003011 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003012 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003013 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
3014 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003015
mflodmancc3d4422017-08-03 08:27:51 -07003016 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003017}
3018
mflodmancc3d4422017-08-03 08:27:51 -07003019TEST_F(VideoStreamEncoderTest,
3020 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02003021 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003022 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07003023
3024 const int kWidth = 1280;
3025 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02003026 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07003027 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003028 video_source_.IncomingCapturedFrame(
3029 CreateFrame(timestamp_ms, kWidth, kHeight));
3030 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003031 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3032 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3033 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3034
3035 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003036 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003037 timestamp_ms += kFrameIntervalMs;
3038 video_source_.IncomingCapturedFrame(
3039 CreateFrame(timestamp_ms, kWidth, kHeight));
3040 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003041 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3042 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3043 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3044
3045 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07003046 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003047 timestamp_ms += kFrameIntervalMs;
3048 video_source_.IncomingCapturedFrame(
3049 CreateFrame(timestamp_ms, kWidth, kHeight));
3050 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003051 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3052 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3053 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3054
Niels Möller4db138e2018-04-19 09:04:13 +02003055 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07003056 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003057
3058 VideoEncoderConfig video_encoder_config;
3059 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3060 // Make format different, to force recreation of encoder.
3061 video_encoder_config.video_format.parameters["foo"] = "foo";
3062 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003063 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003064 timestamp_ms += kFrameIntervalMs;
3065 video_source_.IncomingCapturedFrame(
3066 CreateFrame(timestamp_ms, kWidth, kHeight));
3067 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003068 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3069 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3070 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3071
mflodmancc3d4422017-08-03 08:27:51 -07003072 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003073}
3074
mflodmancc3d4422017-08-03 08:27:51 -07003075TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003076 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003077 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003078 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003079
3080 const int kWidth = 1280;
3081 const int kHeight = 720;
3082 int sequence = 1;
3083
3084 // Enable BALANCED preference, no initial limitation.
3085 test::FrameForwarder source;
3086 video_stream_encoder_->SetSource(&source,
3087 webrtc::DegradationPreference::BALANCED);
3088 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3089 WaitForEncodedFrame(sequence++);
3090 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3091 EXPECT_FALSE(stats.cpu_limited_resolution);
3092 EXPECT_FALSE(stats.cpu_limited_framerate);
3093 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3094
3095 // Trigger CPU overuse, should now adapt down.
3096 video_stream_encoder_->TriggerCpuOveruse();
3097 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3098 WaitForEncodedFrame(sequence++);
3099 stats = stats_proxy_->GetStats();
3100 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3101
3102 // Set new degradation preference should clear restrictions since we changed
3103 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003104 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003105 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3106 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3107 WaitForEncodedFrame(sequence++);
3108 stats = stats_proxy_->GetStats();
3109 EXPECT_FALSE(stats.cpu_limited_resolution);
3110 EXPECT_FALSE(stats.cpu_limited_framerate);
3111 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3112
3113 // Force an input frame rate to be available, or the adaptation call won't
3114 // know what framerate to adapt from.
3115 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3116 mock_stats.input_frame_rate = 30;
3117 stats_proxy_->SetMockStats(mock_stats);
3118 video_stream_encoder_->TriggerCpuOveruse();
3119 stats_proxy_->ResetMockStats();
3120 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3121 WaitForEncodedFrame(sequence++);
3122
3123 // We have now adapted once.
3124 stats = stats_proxy_->GetStats();
3125 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3126
3127 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003128 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3129 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003130 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3131 WaitForEncodedFrame(sequence++);
3132 stats = stats_proxy_->GetStats();
3133 EXPECT_FALSE(stats.cpu_limited_resolution);
3134 EXPECT_FALSE(stats.cpu_limited_framerate);
3135 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3136
3137 video_stream_encoder_->Stop();
3138}
3139
3140TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003141 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003142 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003143 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003144
asapersson0944a802017-04-07 00:57:58 -07003145 const int kWidth = 1280;
3146 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003147 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003148
asaperssonfab67072017-04-04 05:51:49 -07003149 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003150 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003151 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003152 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003153 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003154 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3155
asapersson02465b82017-04-10 01:12:52 -07003156 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003157 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003158 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003159 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003160 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003161 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003162 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003163 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3164
3165 // Set new source with adaptation still enabled.
3166 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003167 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003168 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003169
3170 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003171 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003172 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003173 stats = stats_proxy_->GetStats();
3174 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003175 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003176 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3177
sprangc5d62e22017-04-02 23:53:04 -07003178 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003179 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003180 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003181 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003182 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003183 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003184 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003185 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003186 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003187 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003188 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3189
sprangc5d62e22017-04-02 23:53:04 -07003190 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003191 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003192 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3193 mock_stats.input_frame_rate = 30;
3194 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003195 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003196 stats_proxy_->ResetMockStats();
3197
3198 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003199 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003200 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003201
3202 // Framerate now adapted.
3203 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003204 EXPECT_FALSE(stats.cpu_limited_resolution);
3205 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003206 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3207
3208 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003209 video_stream_encoder_->SetSource(&new_video_source,
3210 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003211 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003212 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003213 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003214
3215 stats = stats_proxy_->GetStats();
3216 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003217 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003218 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3219
3220 // Try to trigger overuse. Should not succeed.
3221 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003222 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003223 stats_proxy_->ResetMockStats();
3224
3225 stats = stats_proxy_->GetStats();
3226 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003227 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003228 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3229
3230 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003231 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003232 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003233 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003234 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003235 stats = stats_proxy_->GetStats();
3236 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003237 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003238 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003239
3240 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003241 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003242 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003243 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003244 stats = stats_proxy_->GetStats();
3245 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003246 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003247 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3248
3249 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003250 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003251 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003252 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003253 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003254 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003255 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003256 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003257 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003258 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003259 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3260
3261 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003262 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003263 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003264 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003265 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003266 stats = stats_proxy_->GetStats();
3267 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003268 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003269 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003270 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003271
mflodmancc3d4422017-08-03 08:27:51 -07003272 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003273}
3274
mflodmancc3d4422017-08-03 08:27:51 -07003275TEST_F(VideoStreamEncoderTest,
3276 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003277 const int kWidth = 1280;
3278 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003279 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003280 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003281
asaperssonfab67072017-04-04 05:51:49 -07003282 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003283 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003284
asaperssonfab67072017-04-04 05:51:49 -07003285 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003286 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003287
asaperssonfab67072017-04-04 05:51:49 -07003288 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003289 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003290
asaperssonfab67072017-04-04 05:51:49 -07003291 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003292 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003293
kthelgason876222f2016-11-29 01:44:11 -08003294 // Expect a scale down.
3295 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003296 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003297
asapersson02465b82017-04-10 01:12:52 -07003298 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003299 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003300 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003301 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003302
asaperssonfab67072017-04-04 05:51:49 -07003303 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003304 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003305 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003306 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003307
asaperssonfab67072017-04-04 05:51:49 -07003308 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003309 EXPECT_EQ(std::numeric_limits<int>::max(),
3310 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003311
asaperssonfab67072017-04-04 05:51:49 -07003312 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003313 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003314 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003315 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003316
asapersson02465b82017-04-10 01:12:52 -07003317 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003318 EXPECT_EQ(std::numeric_limits<int>::max(),
3319 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003320
mflodmancc3d4422017-08-03 08:27:51 -07003321 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003322}
3323
mflodmancc3d4422017-08-03 08:27:51 -07003324TEST_F(VideoStreamEncoderTest,
3325 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003326 const int kWidth = 1280;
3327 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003328 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003329 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003330
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003331 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003332 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003333 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003334 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003335
3336 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003337 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003338 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003339 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3340 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3341
3342 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003343 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003344 EXPECT_THAT(source.sink_wants(),
3345 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003346 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3347 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3348 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3349
3350 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003351 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003352 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3353 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3354 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3355
mflodmancc3d4422017-08-03 08:27:51 -07003356 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003357}
3358
mflodmancc3d4422017-08-03 08:27:51 -07003359TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003360 const int kWidth = 1280;
3361 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003362 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003363 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003364
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003365 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003366 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003367 video_stream_encoder_->SetSource(&source,
3368 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003369 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3370 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003371 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003372
3373 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003374 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003375 EXPECT_THAT(source.sink_wants(),
3376 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003377 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3378 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3379 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3380
3381 // Trigger adapt down for same input resolution, expect no change.
3382 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3383 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003384 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003385 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3386 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3387 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3388
3389 // Trigger adapt down for larger input resolution, expect no change.
3390 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3391 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003392 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003393 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3394 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3395 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3396
mflodmancc3d4422017-08-03 08:27:51 -07003397 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003398}
3399
mflodmancc3d4422017-08-03 08:27:51 -07003400TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003401 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3402 const int kWidth = 640;
3403 const int kHeight = 360;
3404 const int64_t kFrameIntervalMs = 150;
3405 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003406 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003407
3408 // Enable BALANCED preference, no initial limitation.
3409 AdaptingFrameForwarder source(&time_controller_);
3410 source.set_adaptation_enabled(true);
3411 video_stream_encoder_->SetSource(&source,
3412 webrtc::DegradationPreference::BALANCED);
3413
3414 int64_t timestamp_ms = kFrameIntervalMs;
3415 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3416 sink_.WaitForEncodedFrame(kWidth, kHeight);
3417 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3418 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3419 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3420 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3421
3422 // Trigger adapt down, expect reduced fps (640x360@15fps).
3423 video_stream_encoder_->TriggerQualityLow();
3424 timestamp_ms += kFrameIntervalMs;
3425 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3426 sink_.WaitForEncodedFrame(timestamp_ms);
3427 EXPECT_THAT(source.sink_wants(),
3428 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3429 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3430 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3431 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3432
3433 // Source requests 270p, expect reduced resolution (480x270@15fps).
3434 source.OnOutputFormatRequest(480, 270);
3435 timestamp_ms += kFrameIntervalMs;
3436 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3437 WaitForEncodedFrame(480, 270);
3438 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3439
3440 // Trigger adapt down, expect reduced fps (480x270@10fps).
3441 video_stream_encoder_->TriggerQualityLow();
3442 timestamp_ms += kFrameIntervalMs;
3443 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3444 sink_.WaitForEncodedFrame(timestamp_ms);
3445 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3446 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3447 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3448 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3449
3450 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3451 source.OnOutputFormatRequest(320, 180);
3452 timestamp_ms += kFrameIntervalMs;
3453 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3454 WaitForEncodedFrame(320, 180);
3455 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3456
3457 // Trigger adapt down, expect reduced fps (320x180@7fps).
3458 video_stream_encoder_->TriggerQualityLow();
3459 timestamp_ms += kFrameIntervalMs;
3460 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3461 sink_.WaitForEncodedFrame(timestamp_ms);
3462 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3463 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3464 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3465 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3466
3467 // Source requests VGA, expect increased resolution (640x360@7fps).
3468 source.OnOutputFormatRequest(640, 360);
3469 timestamp_ms += kFrameIntervalMs;
3470 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3471 WaitForEncodedFrame(timestamp_ms);
3472 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3473
3474 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3475 video_stream_encoder_->TriggerQualityHigh();
3476 timestamp_ms += kFrameIntervalMs;
3477 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3478 WaitForEncodedFrame(timestamp_ms);
3479 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3480 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3481 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3482 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3483
3484 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3485 video_stream_encoder_->TriggerQualityHigh();
3486 timestamp_ms += kFrameIntervalMs;
3487 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3488 WaitForEncodedFrame(timestamp_ms);
3489 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3490 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3491 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3492 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3493
3494 // Trigger adapt up, expect increased fps (640x360@maxfps).
3495 video_stream_encoder_->TriggerQualityHigh();
3496 timestamp_ms += kFrameIntervalMs;
3497 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3498 WaitForEncodedFrame(timestamp_ms);
3499 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3500 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3501 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3502 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3503
3504 video_stream_encoder_->Stop();
3505}
3506
3507TEST_F(VideoStreamEncoderTest,
3508 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3509 const int kWidth = 1280;
3510 const int kHeight = 720;
3511 const int64_t kFrameIntervalMs = 150;
3512 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003513 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003514
3515 // Enable BALANCED preference, no initial limitation.
3516 AdaptingFrameForwarder source(&time_controller_);
3517 source.set_adaptation_enabled(true);
3518 video_stream_encoder_->SetSource(&source,
3519 webrtc::DegradationPreference::BALANCED);
3520
3521 int64_t timestamp_ms = kFrameIntervalMs;
3522 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3523 sink_.WaitForEncodedFrame(kWidth, kHeight);
3524 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3525 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3526 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3527 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3528
3529 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3530 video_stream_encoder_->TriggerQualityLow();
3531 timestamp_ms += kFrameIntervalMs;
3532 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3533 sink_.WaitForEncodedFrame(timestamp_ms);
3534 EXPECT_THAT(source.sink_wants(),
3535 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3536 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3537 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3538 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3539
3540 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3541 video_stream_encoder_->TriggerQualityLow();
3542 timestamp_ms += kFrameIntervalMs;
3543 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3544 sink_.WaitForEncodedFrame(timestamp_ms);
3545 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3546 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3547 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3548 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3549
3550 // Trigger adapt down, expect reduced fps (640x360@15fps).
3551 video_stream_encoder_->TriggerQualityLow();
3552 timestamp_ms += kFrameIntervalMs;
3553 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3554 WaitForEncodedFrame(timestamp_ms);
3555 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3556 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3557 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3558 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3559
3560 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3561 source.OnOutputFormatRequest(320, 180);
3562 timestamp_ms += kFrameIntervalMs;
3563 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3564 WaitForEncodedFrame(320, 180);
3565 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3566 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3567
3568 // Trigger adapt down, expect reduced fps (320x180@7fps).
3569 video_stream_encoder_->TriggerCpuOveruse();
3570 timestamp_ms += kFrameIntervalMs;
3571 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3572 WaitForEncodedFrame(timestamp_ms);
3573 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3574 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3575 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3576 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3577 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3578 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3579 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3580
3581 // Source requests HD, expect increased resolution (640x360@7fps).
3582 source.OnOutputFormatRequest(1280, 720);
3583 timestamp_ms += kFrameIntervalMs;
3584 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3585 WaitForEncodedFrame(timestamp_ms);
3586 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3587 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3588
3589 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3590 video_stream_encoder_->TriggerCpuUnderuse();
3591 timestamp_ms += kFrameIntervalMs;
3592 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3593 WaitForEncodedFrame(timestamp_ms);
3594 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3595 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3596 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3597 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3598 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3599 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3600 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3601
3602 // Trigger adapt up, expect increased fps (640x360@maxfps).
3603 video_stream_encoder_->TriggerQualityHigh();
3604 video_stream_encoder_->TriggerCpuUnderuse();
3605 timestamp_ms += kFrameIntervalMs;
3606 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3607 WaitForEncodedFrame(timestamp_ms);
3608 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3609 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3610 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3611 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3612 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3613 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3614 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3615
3616 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3617 video_stream_encoder_->TriggerQualityHigh();
3618 video_stream_encoder_->TriggerCpuUnderuse();
3619 timestamp_ms += kFrameIntervalMs;
3620 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3621 WaitForEncodedFrame(timestamp_ms);
3622 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3623 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3624 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3625 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3626 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3627 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3628 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3629
3630 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3631 video_stream_encoder_->TriggerQualityHigh();
3632 video_stream_encoder_->TriggerCpuUnderuse();
3633 timestamp_ms += kFrameIntervalMs;
3634 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3635 WaitForEncodedFrame(timestamp_ms);
3636 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3637 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3638 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3639 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3640 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3641 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3642 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3643
3644 video_stream_encoder_->Stop();
3645}
3646
3647TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003648 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003649 const int kWidth = 1280;
3650 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003651 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003652 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003653
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003654 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003655 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003656 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003657 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003658
3659 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003660 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003661 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003662 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3663 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3664
3665 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003666 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003667 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003668 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3669 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3670
mflodmancc3d4422017-08-03 08:27:51 -07003671 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003672}
3673
mflodmancc3d4422017-08-03 08:27:51 -07003674TEST_F(VideoStreamEncoderTest,
3675 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003676 const int kWidth = 1280;
3677 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003678 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003679 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003680
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003681 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003682 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003683 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003684 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003685
3686 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003687 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003688 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003689 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003690 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3691
3692 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003693 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003694 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003695 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003696 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3697
mflodmancc3d4422017-08-03 08:27:51 -07003698 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003699}
3700
mflodmancc3d4422017-08-03 08:27:51 -07003701TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003702 const int kWidth = 1280;
3703 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003704 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003705 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003706
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003707 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003708 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003709 video_stream_encoder_->SetSource(&source,
3710 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003711
3712 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3713 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003714 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003715 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3716 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3717 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3718
3719 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003720 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003721 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003722 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3723 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3724 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3725
mflodmancc3d4422017-08-03 08:27:51 -07003726 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003727}
3728
mflodmancc3d4422017-08-03 08:27:51 -07003729TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003730 const int kWidth = 1280;
3731 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003732 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003733 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003734
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003735 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003736 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003737 video_stream_encoder_->SetSource(&source,
3738 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003739
3740 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3741 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003742 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003743 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3744 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3745 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3746
3747 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003748 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003749 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003750 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3751 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3752 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3753
mflodmancc3d4422017-08-03 08:27:51 -07003754 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003755}
3756
mflodmancc3d4422017-08-03 08:27:51 -07003757TEST_F(VideoStreamEncoderTest,
3758 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003759 const int kWidth = 1280;
3760 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003761 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003762 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003763
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003764 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003765 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003766 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003767 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003768 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003769
3770 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003771 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003772 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003773 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3774 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3775
3776 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003777 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003778 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003779 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003780 EXPECT_THAT(source.sink_wants(),
3781 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003782 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3783 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3784
3785 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003786 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003787 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003788 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3789 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3790 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3791
mflodmancc3d4422017-08-03 08:27:51 -07003792 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003793}
3794
mflodmancc3d4422017-08-03 08:27:51 -07003795TEST_F(VideoStreamEncoderTest,
3796 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003797 const int kWidth = 1280;
3798 const int kHeight = 720;
3799 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003800 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003801 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003802
3803 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3804 stats.input_frame_rate = kInputFps;
3805 stats_proxy_->SetMockStats(stats);
3806
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003807 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003808 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3809 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003810 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003811
3812 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003813 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003814 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3815 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003816 EXPECT_THAT(video_source_.sink_wants(),
3817 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003818
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003819 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003820 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003821 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003822 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003823 // Give the encoder queue time to process the change in degradation preference
3824 // by waiting for an encoded frame.
3825 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3826 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003827 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003828
3829 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003830 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003831 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3832 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003833 EXPECT_THAT(new_video_source.sink_wants(),
3834 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003835
3836 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003837 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003838 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003839
mflodmancc3d4422017-08-03 08:27:51 -07003840 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003841}
3842
mflodmancc3d4422017-08-03 08:27:51 -07003843TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003844 const int kWidth = 1280;
3845 const int kHeight = 720;
3846 const size_t kNumFrames = 10;
3847
Henrik Boström381d1092020-05-12 18:49:07 +02003848 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003849 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003850
asaperssond0de2952017-04-21 01:47:31 -07003851 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003852 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003853 video_source_.set_adaptation_enabled(true);
3854
3855 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3856 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3857
3858 int downscales = 0;
3859 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003860 video_source_.IncomingCapturedFrame(
3861 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3862 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003863
asaperssonfab67072017-04-04 05:51:49 -07003864 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003865 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003866 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003867 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003868
3869 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3870 ++downscales;
3871
3872 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3873 EXPECT_EQ(downscales,
3874 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3875 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003876 }
mflodmancc3d4422017-08-03 08:27:51 -07003877 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003878}
3879
mflodmancc3d4422017-08-03 08:27:51 -07003880TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003881 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3882 const int kWidth = 1280;
3883 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003884 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003885 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003886
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003887 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003888 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003889 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003890 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003891 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003892
Åsa Persson8c1bf952018-09-13 10:42:19 +02003893 int64_t timestamp_ms = kFrameIntervalMs;
3894 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003895 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003896 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003897 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3898 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3899
3900 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003901 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003902 timestamp_ms += kFrameIntervalMs;
3903 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3904 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003905 EXPECT_THAT(source.sink_wants(),
3906 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003907 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3908 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3909
3910 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003911 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003912 timestamp_ms += kFrameIntervalMs;
3913 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003914 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003915 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003916 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3917 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3918
3919 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003920 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003921 timestamp_ms += kFrameIntervalMs;
3922 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3923 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003924 EXPECT_THAT(source.sink_wants(),
3925 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003926 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3927 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3928
3929 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003930 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003931 timestamp_ms += kFrameIntervalMs;
3932 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003933 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003934 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003935 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3936 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3937
mflodmancc3d4422017-08-03 08:27:51 -07003938 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003939}
3940
mflodmancc3d4422017-08-03 08:27:51 -07003941TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003942 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3943 const int kWidth = 1280;
3944 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003945 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003946 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003947
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003948 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003949 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003950 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003951 video_stream_encoder_->SetSource(&source,
3952 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003953
Åsa Persson8c1bf952018-09-13 10:42:19 +02003954 int64_t timestamp_ms = kFrameIntervalMs;
3955 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003956 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003957 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003958 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3959 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3960
3961 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003962 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003963 timestamp_ms += kFrameIntervalMs;
3964 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3965 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003966 EXPECT_THAT(source.sink_wants(),
3967 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003968 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3969 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3970
3971 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003972 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003973 timestamp_ms += kFrameIntervalMs;
3974 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003975 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003976 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003977 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3978 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3979
3980 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003981 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003982 timestamp_ms += kFrameIntervalMs;
3983 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3984 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003985 EXPECT_THAT(source.sink_wants(),
3986 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003987 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3988 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3989
3990 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003991 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003992 timestamp_ms += kFrameIntervalMs;
3993 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003994 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003995 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003996 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3997 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3998
mflodmancc3d4422017-08-03 08:27:51 -07003999 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004000}
4001
Sergey Silkin41c650b2019-10-14 13:12:19 +02004002TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
4003 fake_encoder_.SetResolutionBitrateLimits(
4004 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4005
Henrik Boström381d1092020-05-12 18:49:07 +02004006 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004007 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4008 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4009 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4010 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004011
4012 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004013 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004014 source.set_adaptation_enabled(true);
4015 video_stream_encoder_->SetSource(
4016 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4017
4018 // Insert 720p frame.
4019 int64_t timestamp_ms = kFrameIntervalMs;
4020 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4021 WaitForEncodedFrame(1280, 720);
4022
4023 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02004024 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004025 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4026 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4027 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4028 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004029 video_stream_encoder_->TriggerQualityLow();
4030
4031 // Insert 720p frame. It should be downscaled and encoded.
4032 timestamp_ms += kFrameIntervalMs;
4033 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4034 WaitForEncodedFrame(960, 540);
4035
4036 // Trigger adapt up. Higher resolution should not be requested duo to lack
4037 // of bitrate.
4038 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004039 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02004040
4041 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02004042 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004043 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4044 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4045 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4046 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004047
4048 // Trigger adapt up. Higher resolution should be requested.
4049 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004050 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02004051
4052 video_stream_encoder_->Stop();
4053}
4054
4055TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
4056 fake_encoder_.SetResolutionBitrateLimits(
4057 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4058
4059 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02004060 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004061 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4062 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4063 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4064 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004065
4066 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004067 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004068 source.set_adaptation_enabled(true);
4069 video_stream_encoder_->SetSource(
4070 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4071
4072 // Insert 720p frame. It should be dropped and lower resolution should be
4073 // requested.
4074 int64_t timestamp_ms = kFrameIntervalMs;
4075 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4076 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004077 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004078
4079 // Insert 720p frame. It should be downscaled and encoded.
4080 timestamp_ms += kFrameIntervalMs;
4081 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4082 WaitForEncodedFrame(960, 540);
4083
4084 video_stream_encoder_->Stop();
4085}
4086
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004087class BalancedDegradationTest : public VideoStreamEncoderTest {
4088 protected:
4089 void SetupTest() {
4090 // Reset encoder for field trials to take effect.
4091 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02004092 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004093
4094 // Enable BALANCED preference.
4095 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02004096 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4097 }
4098
Asa Persson606d3cb2021-10-04 10:07:11 +02004099 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02004100 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004101 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004102 }
4103
Åsa Persson45b176f2019-09-30 11:19:05 +02004104 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004105 timestamp_ms_ += kFrameIntervalMs;
4106 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02004107 }
4108
4109 void InsertFrameAndWaitForEncoded() {
4110 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004111 sink_.WaitForEncodedFrame(timestamp_ms_);
4112 }
4113
4114 const int kWidth = 640; // pixels:640x360=230400
4115 const int kHeight = 360;
4116 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4117 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004118 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004119};
4120
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004121TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004122 test::ScopedFieldTrials field_trials(
4123 "WebRTC-Video-BalancedDegradationSettings/"
4124 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4125 SetupTest();
4126
4127 // Force input frame rate.
4128 const int kInputFps = 24;
4129 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4130 stats.input_frame_rate = kInputFps;
4131 stats_proxy_->SetMockStats(stats);
4132
Åsa Persson45b176f2019-09-30 11:19:05 +02004133 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004134 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004135
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004136 // Trigger adapt down, expect scaled down framerate and resolution,
4137 // since Fps diff (input-requested:0) < threshold.
4138 video_stream_encoder_->TriggerQualityLow();
4139 EXPECT_THAT(source_.sink_wants(),
4140 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004141
4142 video_stream_encoder_->Stop();
4143}
4144
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004145TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004146 test::ScopedFieldTrials field_trials(
4147 "WebRTC-Video-BalancedDegradationSettings/"
4148 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4149 SetupTest();
4150
4151 // Force input frame rate.
4152 const int kInputFps = 25;
4153 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4154 stats.input_frame_rate = kInputFps;
4155 stats_proxy_->SetMockStats(stats);
4156
Åsa Persson45b176f2019-09-30 11:19:05 +02004157 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004158 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004159
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004160 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4161 // Fps diff (input-requested:1) == threshold.
4162 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004163 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004164
4165 video_stream_encoder_->Stop();
4166}
4167
4168TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
4169 test::ScopedFieldTrials field_trials(
4170 "WebRTC-Video-BalancedDegradationSettings/"
4171 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4172 SetupTest();
4173
4174 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4175
Åsa Persson45b176f2019-09-30 11:19:05 +02004176 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004177 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004178
4179 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4180 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004181 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004182
4183 video_stream_encoder_->Stop();
4184}
4185
Åsa Perssonccfb3402019-09-25 15:13:04 +02004186TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004187 test::ScopedFieldTrials field_trials(
Åsa Persson1b247f12019-08-14 17:26:39 +02004188 "WebRTC-Video-BalancedDegradationSettings/"
4189 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004190 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004191
Asa Persson606d3cb2021-10-04 10:07:11 +02004192 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4193 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4194 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004195
Åsa Persson45b176f2019-09-30 11:19:05 +02004196 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004197 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004198 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4199
4200 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4201 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004202 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004203 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004204 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4205
4206 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4207 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004208 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004209 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004210 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4211
Åsa Persson30ab0152019-08-27 12:22:33 +02004212 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4213 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004214 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004215 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004216 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004217 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4218
4219 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004220 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004221 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004222 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004223
Åsa Persson30ab0152019-08-27 12:22:33 +02004224 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004225 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004226 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004227 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004228 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004229 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4230
4231 video_stream_encoder_->Stop();
4232}
4233
Åsa Perssonccfb3402019-09-25 15:13:04 +02004234TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004235 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
4236 test::ScopedFieldTrials field_trials(
4237 "WebRTC-Video-BalancedDegradationSettings/"
4238 "pixels:57600|129600|230400,fps:7|24|24/");
4239 SetupTest();
Asa Persson606d3cb2021-10-04 10:07:11 +02004240 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 11:19:05 +02004241
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004242 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004243
4244 // Insert frame, expect scaled down:
4245 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4246 InsertFrame();
4247 EXPECT_FALSE(WaitForFrame(1000));
4248 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4249 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4250
4251 // Insert frame, expect scaled down:
4252 // resolution (320x180@24fps).
4253 InsertFrame();
4254 EXPECT_FALSE(WaitForFrame(1000));
4255 EXPECT_LT(source_.sink_wants().max_pixel_count,
4256 source_.last_wants().max_pixel_count);
4257 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4258
4259 // Frame should not be dropped (min pixels per frame reached).
4260 InsertFrameAndWaitForEncoded();
4261
4262 video_stream_encoder_->Stop();
4263}
4264
4265TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004266 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004267 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004268 "WebRTC-Video-BalancedDegradationSettings/"
4269 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004270 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004271
Asa Persson606d3cb2021-10-04 10:07:11 +02004272 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4273 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4274 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004275
Åsa Persson45b176f2019-09-30 11:19:05 +02004276 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004277 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004278 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4279
4280 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4281 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004282 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004283 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004284 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4285
4286 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4287 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004288 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004289 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004290 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4291
4292 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4293 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004294 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004295 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004296 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4297
Åsa Persson30ab0152019-08-27 12:22:33 +02004298 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4299 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004300 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004301 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004302 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4303
4304 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4305 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004306 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004307 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4308
4309 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004310 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004311 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004312 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004313 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004314 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4315
4316 video_stream_encoder_->Stop();
4317}
4318
Åsa Perssonccfb3402019-09-25 15:13:04 +02004319TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004320 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004321 test::ScopedFieldTrials field_trials(
Åsa Persson30ab0152019-08-27 12:22:33 +02004322 "WebRTC-Video-BalancedDegradationSettings/"
4323 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004324 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004325
Asa Persson606d3cb2021-10-04 10:07:11 +02004326 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4327 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4328 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4329 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4330 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004331
Åsa Persson45b176f2019-09-30 11:19:05 +02004332 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004333 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004334 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4335
4336 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4337 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004338 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004339 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004340 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4341
4342 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4343 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004344 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004345 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004346 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4347
4348 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4349 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004350 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004351 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004352 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4353
4354 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4355 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004356 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004357 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4358
4359 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004360 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004361 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004362 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004363 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004364 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4365
4366 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004367 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004368 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004369 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004370 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4371
4372 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004373 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004374 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004375 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004376 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004377 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4378
Åsa Persson1b247f12019-08-14 17:26:39 +02004379 video_stream_encoder_->Stop();
4380}
4381
mflodmancc3d4422017-08-03 08:27:51 -07004382TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004383 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4384 const int kWidth = 1280;
4385 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004386 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004387 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004388
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004389 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004390 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004391 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004392 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004393 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004394
Åsa Persson8c1bf952018-09-13 10:42:19 +02004395 int64_t timestamp_ms = kFrameIntervalMs;
4396 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004397 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004398 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004399 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4400 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4401 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4402 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4403
4404 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004405 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004406 timestamp_ms += kFrameIntervalMs;
4407 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4408 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004409 EXPECT_THAT(source.sink_wants(),
4410 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004411 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4412 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4413 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4414 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4415
4416 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004417 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004418 timestamp_ms += kFrameIntervalMs;
4419 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4420 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004421 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004422 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4423 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4424 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4425 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4426
Jonathan Yubc771b72017-12-08 17:04:29 -08004427 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004428 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004429 timestamp_ms += kFrameIntervalMs;
4430 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4431 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004432 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004433 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4434 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004435 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004436 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4437
Jonathan Yubc771b72017-12-08 17:04:29 -08004438 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004439 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004440 timestamp_ms += kFrameIntervalMs;
4441 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4442 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004443 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004444 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004445 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4446 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4447 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4448 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4449
Jonathan Yubc771b72017-12-08 17:04:29 -08004450 // Trigger quality adapt down, expect no change (min resolution reached).
4451 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004452 timestamp_ms += kFrameIntervalMs;
4453 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4454 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004455 EXPECT_THAT(source.sink_wants(), FpsMax());
4456 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004457 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4458 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4459 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4460 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4461
Evan Shrubsole64469032020-06-11 10:45:29 +02004462 // Trigger quality adapt up, expect upscaled resolution (480x270).
4463 video_stream_encoder_->TriggerQualityHigh();
4464 timestamp_ms += kFrameIntervalMs;
4465 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4466 WaitForEncodedFrame(timestamp_ms);
4467 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4468 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4469 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4470 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4471 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4472
4473 // Trigger quality and cpu adapt up since both are most limited, expect
4474 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004475 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004476 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004477 timestamp_ms += kFrameIntervalMs;
4478 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4479 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004480 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004481 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4482 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4483 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004484 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004485
Evan Shrubsole64469032020-06-11 10:45:29 +02004486 // Trigger quality and cpu adapt up since both are most limited, expect
4487 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004488 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004489 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004490 timestamp_ms += kFrameIntervalMs;
4491 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4492 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004493 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004494 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004495 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004496 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004497 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4498 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004499
Evan Shrubsole64469032020-06-11 10:45:29 +02004500 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4501 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004502 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004503 timestamp_ms += kFrameIntervalMs;
4504 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4505 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004506 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004507 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4508 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004509 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004510 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004511
4512 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004513 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004514 timestamp_ms += kFrameIntervalMs;
4515 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004516 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004517 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004518 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004519 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4520 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004521 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004522 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004523
mflodmancc3d4422017-08-03 08:27:51 -07004524 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004525}
4526
mflodmancc3d4422017-08-03 08:27:51 -07004527TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004528 const int kWidth = 640;
4529 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004530
Henrik Boström381d1092020-05-12 18:49:07 +02004531 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004532 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004533
perkj803d97f2016-11-01 11:45:46 -07004534 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004535 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004536 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004537 }
4538
mflodmancc3d4422017-08-03 08:27:51 -07004539 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004540 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004541 video_source_.IncomingCapturedFrame(CreateFrame(
4542 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004543 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004544 }
4545
mflodmancc3d4422017-08-03 08:27:51 -07004546 video_stream_encoder_->Stop();
4547 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004548 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004549
Ying Wangef3998f2019-12-09 13:06:53 +01004550 EXPECT_METRIC_EQ(
4551 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4552 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004553 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4554}
4555
mflodmancc3d4422017-08-03 08:27:51 -07004556TEST_F(VideoStreamEncoderTest,
4557 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004558 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004559 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004560 const int kWidth = 640;
4561 const int kHeight = 360;
4562
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004563 video_stream_encoder_->SetSource(&video_source_,
4564 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004565
4566 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4567 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004568 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004569 }
4570
mflodmancc3d4422017-08-03 08:27:51 -07004571 video_stream_encoder_->Stop();
4572 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004573 stats_proxy_.reset();
4574
4575 EXPECT_EQ(0,
4576 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4577}
4578
Per Kjellanderdcef6412020-10-07 15:09:05 +02004579TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4580 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004581 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004582 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004583
4584 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004585 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 10:07:11 +02004586 SimulcastRateAllocator(fake_encoder_.config())
4587 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004588 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004589
Henrik Boström381d1092020-05-12 18:49:07 +02004590 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004591 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004592
sprang57c2fff2017-01-16 06:24:02 -08004593 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004594 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4595 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004596 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4597 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4598
Erik Språngd7329ca2019-02-21 21:19:53 +01004599 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004600 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004601 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004602
Per Kjellanderdcef6412020-10-07 15:09:05 +02004603 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004604 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004605 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4606 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004607 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004608 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004609
Per Kjellanderdcef6412020-10-07 15:09:05 +02004610 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004611 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004612 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004613 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004614 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4615 WaitForEncodedFrame(CurrentTimeMs());
4616 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004617 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004618 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004619
mflodmancc3d4422017-08-03 08:27:51 -07004620 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004621}
4622
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004623TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004624 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004625 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004626 kVideoLayersAllocation);
4627
4628 const int kDefaultFps = 30;
4629
4630 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004631 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004632
4633 video_source_.IncomingCapturedFrame(
4634 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4635 WaitForEncodedFrame(CurrentTimeMs());
4636 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4637 VideoLayersAllocation last_layer_allocation =
4638 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004639 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004640 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4641
4642 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004643 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004644 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 10:07:11 +02004645 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 17:53:22 +02004646 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4647
Erik Språng9d69cbe2020-10-22 17:44:42 +02004648 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004649 int number_of_layers_allocation = 1;
4650 const int64_t start_time_ms = CurrentTimeMs();
4651 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4652 video_source_.IncomingCapturedFrame(
4653 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4654 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004655 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4656 number_of_layers_allocation = sink_.number_of_layers_allocations();
4657 VideoLayersAllocation new_allocation =
4658 sink_.GetLastVideoLayersAllocation();
4659 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4660 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4661 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4662 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4663 .target_bitrate_per_temporal_layer,
4664 last_layer_allocation.active_spatial_layers[0]
4665 .target_bitrate_per_temporal_layer);
4666 last_layer_allocation = new_allocation;
4667 }
4668 }
4669 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4670 video_stream_encoder_->Stop();
4671}
4672
4673TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004674 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004675 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4676 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4677 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004678 VideoEncoderConfig video_encoder_config;
4679 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4680 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004681 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004682 video_encoder_config.content_type =
4683 VideoEncoderConfig::ContentType::kRealtimeVideo;
4684 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004685 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004686 VideoEncoder::GetDefaultVp8Settings());
4687 for (auto& layer : video_encoder_config.simulcast_layers) {
4688 layer.num_temporal_layers = 2;
4689 }
4690 // Simulcast layers are used for enabling/disabling streams.
4691 video_encoder_config.simulcast_layers[0].active = true;
4692 video_encoder_config.simulcast_layers[1].active = false;
4693 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004694 ConfigureEncoder(std::move(video_encoder_config),
4695 VideoStreamEncoder::BitrateAllocationCallbackType::
4696 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004697
4698 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004699 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004700
4701 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4702 WaitForEncodedFrame(CurrentTimeMs());
4703 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4704 VideoLayersAllocation last_layer_allocation =
4705 sink_.GetLastVideoLayersAllocation();
4706
4707 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4708 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4709 .target_bitrate_per_temporal_layer,
4710 SizeIs(2));
4711 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4712 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4713 video_stream_encoder_->Stop();
4714}
4715
4716TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004717 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004718 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4719 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4720 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004721 VideoEncoderConfig video_encoder_config;
4722 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4723 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004724 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004725 video_encoder_config.content_type =
4726 VideoEncoderConfig::ContentType::kRealtimeVideo;
4727 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004728 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004729 VideoEncoder::GetDefaultVp8Settings());
4730 for (auto& layer : video_encoder_config.simulcast_layers) {
4731 layer.num_temporal_layers = 2;
4732 }
4733 // Simulcast layers are used for enabling/disabling streams.
4734 video_encoder_config.simulcast_layers[0].active = true;
4735 video_encoder_config.simulcast_layers[1].active = false;
4736 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004737 ConfigureEncoder(std::move(video_encoder_config),
4738 VideoStreamEncoder::BitrateAllocationCallbackType::
4739 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004740
4741 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004742 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004743
4744 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4745 WaitForEncodedFrame(CurrentTimeMs());
4746 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4747 VideoLayersAllocation last_layer_allocation =
4748 sink_.GetLastVideoLayersAllocation();
4749
4750 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4751 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4752 .target_bitrate_per_temporal_layer,
4753 SizeIs(2));
4754 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4755
4756 video_stream_encoder_->Stop();
4757}
4758
4759TEST_F(VideoStreamEncoderTest,
4760 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4761 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4762 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004763 VideoEncoderConfig video_encoder_config;
4764 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4765 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004766 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004767 video_encoder_config.content_type =
4768 VideoEncoderConfig::ContentType::kRealtimeVideo;
4769 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4770 vp9_settings.numberOfSpatialLayers = 2;
4771 vp9_settings.numberOfTemporalLayers = 2;
4772 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4773 vp9_settings.automaticResizeOn = false;
4774 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004775 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004776 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004777 ConfigureEncoder(std::move(video_encoder_config),
4778 VideoStreamEncoder::BitrateAllocationCallbackType::
4779 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004780
4781 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004782 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004783
4784 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4785 WaitForEncodedFrame(CurrentTimeMs());
4786 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4787 VideoLayersAllocation last_layer_allocation =
4788 sink_.GetLastVideoLayersAllocation();
4789
4790 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4791 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4792 .target_bitrate_per_temporal_layer,
4793 SizeIs(2));
4794 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4795 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4796 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4797 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4798 .target_bitrate_per_temporal_layer,
4799 SizeIs(2));
4800 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4801 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4802 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4803
4804 // Since full SVC is used, expect the top layer to utilize the full target
4805 // rate.
4806 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4807 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004808 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004809 video_stream_encoder_->Stop();
4810}
4811
4812TEST_F(VideoStreamEncoderTest,
4813 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4814 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4815 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004816 VideoEncoderConfig video_encoder_config;
4817 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4818 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004819 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004820 video_encoder_config.content_type =
4821 VideoEncoderConfig::ContentType::kRealtimeVideo;
4822 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4823 vp9_settings.numberOfSpatialLayers = 2;
4824 vp9_settings.numberOfTemporalLayers = 2;
4825 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4826 vp9_settings.automaticResizeOn = false;
4827 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004828 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004829 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004830 ConfigureEncoder(std::move(video_encoder_config),
4831 VideoStreamEncoder::BitrateAllocationCallbackType::
4832 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004833
4834 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004835 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004836
4837 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4838 WaitForEncodedFrame(CurrentTimeMs());
4839 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4840 VideoLayersAllocation last_layer_allocation =
4841 sink_.GetLastVideoLayersAllocation();
4842
4843 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4844 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4845 .target_bitrate_per_temporal_layer,
4846 SizeIs(1));
4847 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4848 .target_bitrate_per_temporal_layer,
4849 SizeIs(1));
4850 // Since full SVC is used, expect the top layer to utilize the full target
4851 // rate.
4852 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4853 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004854 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004855 video_stream_encoder_->Stop();
4856}
4857
4858TEST_F(VideoStreamEncoderTest,
4859 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4860 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4861 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004862 VideoEncoderConfig video_encoder_config;
4863 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4864 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004865 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004866 video_encoder_config.content_type =
4867 VideoEncoderConfig::ContentType::kRealtimeVideo;
4868 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4869 vp9_settings.numberOfSpatialLayers = 2;
4870 vp9_settings.numberOfTemporalLayers = 2;
4871 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4872 vp9_settings.automaticResizeOn = false;
4873 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004874 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004875 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004876 ConfigureEncoder(std::move(video_encoder_config),
4877 VideoStreamEncoder::BitrateAllocationCallbackType::
4878 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004879
4880 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004881 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004882
4883 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4884 WaitForEncodedFrame(CurrentTimeMs());
4885 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4886 VideoLayersAllocation last_layer_allocation =
4887 sink_.GetLastVideoLayersAllocation();
4888
4889 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4890 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4891 .target_bitrate_per_temporal_layer,
4892 SizeIs(2));
4893 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4894 .target_bitrate_per_temporal_layer,
4895 SizeIs(2));
4896 // Since KSVC is, spatial layers are independend except on key frames.
4897 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4898 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004899 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004900 video_stream_encoder_->Stop();
4901}
4902
4903TEST_F(VideoStreamEncoderTest,
4904 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4905 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4906 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4907 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004908 VideoEncoderConfig video_encoder_config;
4909 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4910 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004911 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004912 video_encoder_config.content_type =
4913 VideoEncoderConfig::ContentType::kRealtimeVideo;
4914 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4915 vp9_settings.numberOfSpatialLayers = 3;
4916 vp9_settings.numberOfTemporalLayers = 2;
4917 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4918 vp9_settings.automaticResizeOn = false;
4919 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004920 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004921 vp9_settings);
4922 // Simulcast layers are used for enabling/disabling streams.
4923 video_encoder_config.simulcast_layers.resize(3);
4924 video_encoder_config.simulcast_layers[0].active = false;
4925 video_encoder_config.simulcast_layers[1].active = true;
4926 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004927 ConfigureEncoder(std::move(video_encoder_config),
4928 VideoStreamEncoder::BitrateAllocationCallbackType::
4929 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004930
4931 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004932 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004933
4934 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4935 WaitForEncodedFrame(CurrentTimeMs());
4936 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4937 VideoLayersAllocation last_layer_allocation =
4938 sink_.GetLastVideoLayersAllocation();
4939
4940 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4941 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4942 .target_bitrate_per_temporal_layer,
4943 SizeIs(2));
4944 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4945 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4946
4947 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4948 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4949 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4950 .target_bitrate_per_temporal_layer,
4951 SizeIs(2));
4952 // Since full SVC is used, expect the top layer to utilize the full target
4953 // rate.
4954 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4955 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004956 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004957 video_stream_encoder_->Stop();
4958}
4959
4960TEST_F(VideoStreamEncoderTest,
4961 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4962 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4963 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4964 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004965 VideoEncoderConfig video_encoder_config;
4966 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4967 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004968 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004969 video_encoder_config.content_type =
4970 VideoEncoderConfig::ContentType::kRealtimeVideo;
4971 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4972 vp9_settings.numberOfSpatialLayers = 3;
4973 vp9_settings.numberOfTemporalLayers = 2;
4974 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4975 vp9_settings.automaticResizeOn = false;
4976 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004977 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004978 vp9_settings);
4979 // Simulcast layers are used for enabling/disabling streams.
4980 video_encoder_config.simulcast_layers.resize(3);
4981 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004982 ConfigureEncoder(std::move(video_encoder_config),
4983 VideoStreamEncoder::BitrateAllocationCallbackType::
4984 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004985
4986 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004987 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004988
4989 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4990 WaitForEncodedFrame(CurrentTimeMs());
4991 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4992 VideoLayersAllocation last_layer_allocation =
4993 sink_.GetLastVideoLayersAllocation();
4994
4995 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4996 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4997 .target_bitrate_per_temporal_layer,
4998 SizeIs(2));
4999 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
5000 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5001
5002 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
5003 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
5004 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
5005 .target_bitrate_per_temporal_layer,
5006 SizeIs(2));
5007 video_stream_encoder_->Stop();
5008}
5009
5010TEST_F(VideoStreamEncoderTest,
5011 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
5012 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5013 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5014 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005015 VideoEncoderConfig video_encoder_config;
5016 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5017 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005018 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005019 video_encoder_config.content_type =
5020 VideoEncoderConfig::ContentType::kRealtimeVideo;
5021 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5022 vp9_settings.numberOfSpatialLayers = 3;
5023 vp9_settings.numberOfTemporalLayers = 2;
5024 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5025 vp9_settings.automaticResizeOn = false;
5026 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005027 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005028 vp9_settings);
5029 // Simulcast layers are used for enabling/disabling streams.
5030 video_encoder_config.simulcast_layers.resize(3);
5031 video_encoder_config.simulcast_layers[0].active = false;
5032 video_encoder_config.simulcast_layers[1].active = false;
5033 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005034 ConfigureEncoder(std::move(video_encoder_config),
5035 VideoStreamEncoder::BitrateAllocationCallbackType::
5036 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005037
5038 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005039 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005040
5041 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5042 WaitForEncodedFrame(CurrentTimeMs());
5043 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5044 VideoLayersAllocation last_layer_allocation =
5045 sink_.GetLastVideoLayersAllocation();
5046
5047 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5048 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5049 .target_bitrate_per_temporal_layer,
5050 SizeIs(2));
5051 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5052 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5053 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5054 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02005055 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005056 video_stream_encoder_->Stop();
5057}
5058
5059TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5060 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005061 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005062 kVideoLayersAllocation);
5063 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005064 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005065
5066 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5067 WaitForEncodedFrame(CurrentTimeMs());
5068 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5069 VideoLayersAllocation last_layer_allocation =
5070 sink_.GetLastVideoLayersAllocation();
5071
5072 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5073 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5074 .target_bitrate_per_temporal_layer,
5075 SizeIs(1));
5076 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5077 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005078 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005079 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5080 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5081 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5082 video_stream_encoder_->Stop();
5083}
5084
5085TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02005086 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5087 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005088 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02005089 kVideoLayersAllocation);
5090
5091 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005092 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005093
5094 video_source_.IncomingCapturedFrame(
5095 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5096 WaitForEncodedFrame(CurrentTimeMs());
5097 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5098 VideoLayersAllocation last_layer_allocation =
5099 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02005100 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02005101 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5102 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5103 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005104 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 17:53:22 +02005105
5106 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005107 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5108 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005109 video_source_.IncomingCapturedFrame(
5110 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5111 WaitForEncodedFrame(CurrentTimeMs());
5112
5113 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5114 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5115 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5116 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5117 .target_bitrate_per_temporal_layer[0],
5118 DataRate::Zero());
5119
5120 video_stream_encoder_->Stop();
5121}
5122
Per Kjellander4190ce92020-12-15 17:24:55 +01005123TEST_F(VideoStreamEncoderTest,
5124 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5125 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005126 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01005127 kVideoLayersAllocation);
5128
5129 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005130 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5131 0, 0, 0);
Per Kjellander4190ce92020-12-15 17:24:55 +01005132
5133 video_source_.IncomingCapturedFrame(
5134 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5135 WaitForEncodedFrame(CurrentTimeMs());
5136 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5137 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5138 SizeIs(2));
5139 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5140 codec_width_);
5141 EXPECT_EQ(
5142 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5143 codec_height_);
5144
5145 video_source_.IncomingCapturedFrame(
5146 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5147 WaitForEncodedFrame(CurrentTimeMs());
5148 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5149 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5150 SizeIs(2));
5151 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5152 codec_width_ / 2);
5153 EXPECT_EQ(
5154 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5155 codec_height_ / 2);
5156
5157 video_stream_encoder_->Stop();
5158}
5159
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005160TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5161 // 2 TLs configured, temporal layers supported by encoder.
5162 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005163 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005164 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005165 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005166 fake_encoder_.SetTemporalLayersSupported(0, true);
5167
5168 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 10:07:11 +02005169 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005170 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005171 kNumTemporalLayers, /*temporal_id*/ 0,
5172 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005173 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005174 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005175 kNumTemporalLayers, /*temporal_id*/ 1,
5176 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005177 VideoBitrateAllocation expected_bitrate;
5178 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5179 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5180
5181 VerifyAllocatedBitrate(expected_bitrate);
5182 video_stream_encoder_->Stop();
5183}
5184
5185TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5186 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005187 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005188 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005189 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005190 fake_encoder_.SetTemporalLayersSupported(0, false);
5191
5192 // Temporal layers not supported by the encoder.
5193 // Total bitrate should be at ti:0.
5194 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005195 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005196
5197 VerifyAllocatedBitrate(expected_bitrate);
5198 video_stream_encoder_->Stop();
5199}
5200
5201TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Per Kjellanderdcef6412020-10-07 15:09:05 +02005202 webrtc::test::ScopedFieldTrials field_trials(
5203 "WebRTC-Video-QualityScalerSettings/"
5204 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5205 // Reset encoder for field trials to take effect.
5206 ConfigureEncoder(video_encoder_config_.Copy());
5207
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005208 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005209 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005210 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005211 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005212 fake_encoder_.SetTemporalLayersSupported(0, true);
5213 fake_encoder_.SetTemporalLayersSupported(1, false);
5214
5215 const int kS0Bps = 150000;
5216 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005217 kS0Bps *
5218 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5219 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005220 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005221 kS0Bps *
5222 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5223 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005224 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005225 // Temporal layers not supported by si:1.
5226 VideoBitrateAllocation expected_bitrate;
5227 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5228 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5229 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5230
5231 VerifyAllocatedBitrate(expected_bitrate);
5232 video_stream_encoder_->Stop();
5233}
5234
Niels Möller7dc26b72017-12-06 10:27:48 +01005235TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5236 const int kFrameWidth = 1280;
5237 const int kFrameHeight = 720;
5238 const int kFramerate = 24;
5239
Henrik Boström381d1092020-05-12 18:49:07 +02005240 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005241 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005242 test::FrameForwarder source;
5243 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005244 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005245
5246 // Insert a single frame, triggering initial configuration.
5247 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5248 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5249
5250 EXPECT_EQ(
5251 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5252 kDefaultFramerate);
5253
5254 // Trigger reconfigure encoder (without resetting the entire instance).
5255 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005256 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5257 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005258 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005259 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005260 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005261 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5262
5263 // Detector should be updated with fps limit from codec config.
5264 EXPECT_EQ(
5265 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5266 kFramerate);
5267
5268 // Trigger overuse, max framerate should be reduced.
5269 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5270 stats.input_frame_rate = kFramerate;
5271 stats_proxy_->SetMockStats(stats);
5272 video_stream_encoder_->TriggerCpuOveruse();
5273 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5274 int adapted_framerate =
5275 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5276 EXPECT_LT(adapted_framerate, kFramerate);
5277
5278 // Trigger underuse, max framerate should go back to codec configured fps.
5279 // Set extra low fps, to make sure it's actually reset, not just incremented.
5280 stats = stats_proxy_->GetStats();
5281 stats.input_frame_rate = adapted_framerate / 2;
5282 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005283 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005284 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5285 EXPECT_EQ(
5286 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5287 kFramerate);
5288
5289 video_stream_encoder_->Stop();
5290}
5291
5292TEST_F(VideoStreamEncoderTest,
5293 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5294 const int kFrameWidth = 1280;
5295 const int kFrameHeight = 720;
5296 const int kLowFramerate = 15;
5297 const int kHighFramerate = 25;
5298
Henrik Boström381d1092020-05-12 18:49:07 +02005299 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005300 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005301 test::FrameForwarder source;
5302 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005303 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005304
5305 // Trigger initial configuration.
5306 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005307 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5308 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005309 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005310 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005311 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005312 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005313 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5314
5315 EXPECT_EQ(
5316 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5317 kLowFramerate);
5318
5319 // Trigger overuse, max framerate should be reduced.
5320 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5321 stats.input_frame_rate = kLowFramerate;
5322 stats_proxy_->SetMockStats(stats);
5323 video_stream_encoder_->TriggerCpuOveruse();
5324 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5325 int adapted_framerate =
5326 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5327 EXPECT_LT(adapted_framerate, kLowFramerate);
5328
5329 // Reconfigure the encoder with a new (higher max framerate), max fps should
5330 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005331 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005332 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5333 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005334 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005335 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5336
5337 EXPECT_EQ(
5338 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5339 adapted_framerate);
5340
5341 // Trigger underuse, max framerate should go back to codec configured fps.
5342 stats = stats_proxy_->GetStats();
5343 stats.input_frame_rate = adapted_framerate;
5344 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005345 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005346 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5347 EXPECT_EQ(
5348 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5349 kHighFramerate);
5350
5351 video_stream_encoder_->Stop();
5352}
5353
mflodmancc3d4422017-08-03 08:27:51 -07005354TEST_F(VideoStreamEncoderTest,
5355 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005356 const int kFrameWidth = 1280;
5357 const int kFrameHeight = 720;
5358 const int kFramerate = 24;
5359
Henrik Boström381d1092020-05-12 18:49:07 +02005360 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005361 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005362 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005363 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005364 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005365
5366 // Trigger initial configuration.
5367 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005368 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5369 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005370 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 04:21:07 -07005371 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005372 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005373 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005374 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005375
Niels Möller7dc26b72017-12-06 10:27:48 +01005376 EXPECT_EQ(
5377 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5378 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005379
5380 // Trigger overuse, max framerate should be reduced.
5381 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5382 stats.input_frame_rate = kFramerate;
5383 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005384 video_stream_encoder_->TriggerCpuOveruse();
5385 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005386 int adapted_framerate =
5387 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005388 EXPECT_LT(adapted_framerate, kFramerate);
5389
5390 // Change degradation preference to not enable framerate scaling. Target
5391 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005392 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005393 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005394 EXPECT_EQ(
5395 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5396 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005397
mflodmancc3d4422017-08-03 08:27:51 -07005398 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005399}
5400
mflodmancc3d4422017-08-03 08:27:51 -07005401TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005402 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005403 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005404 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5405 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5406 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005407 const int kWidth = 640;
5408 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005409
asaperssonfab67072017-04-04 05:51:49 -07005410 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005411
5412 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005413 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005414
5415 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005416 EXPECT_TRUE_WAIT(
5417 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005418
sprangc5d62e22017-04-02 23:53:04 -07005419 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005420
asaperssonfab67072017-04-04 05:51:49 -07005421 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005422 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005423 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005424
5425 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005426 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005427
Henrik Boström2671dac2020-05-19 16:29:09 +02005428 EXPECT_TRUE_WAIT(
5429 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005430
mflodmancc3d4422017-08-03 08:27:51 -07005431 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005432}
5433
mflodmancc3d4422017-08-03 08:27:51 -07005434TEST_F(VideoStreamEncoderTest,
5435 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005436 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005437 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005438 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5439 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5440 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005441 const int kWidth = 640;
5442 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005443
5444 // We expect the n initial frames to get dropped.
5445 int i;
5446 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005447 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005448 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005449 }
5450 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005451 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005452 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005453
5454 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005455 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005456
mflodmancc3d4422017-08-03 08:27:51 -07005457 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005458}
5459
mflodmancc3d4422017-08-03 08:27:51 -07005460TEST_F(VideoStreamEncoderTest,
5461 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005462 const int kWidth = 640;
5463 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005464 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005465 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005466
5467 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005468 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005469 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005470
asaperssonfab67072017-04-04 05:51:49 -07005471 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005472 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005473 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005474
mflodmancc3d4422017-08-03 08:27:51 -07005475 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005476}
5477
mflodmancc3d4422017-08-03 08:27:51 -07005478TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005479 const int kWidth = 640;
5480 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005481 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005482
5483 VideoEncoderConfig video_encoder_config;
5484 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5485 // Make format different, to force recreation of encoder.
5486 video_encoder_config.video_format.parameters["foo"] = "foo";
5487 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005488 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005489 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005490 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005491
kthelgasonb83797b2017-02-14 11:57:25 -08005492 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005493 video_stream_encoder_->SetSource(&video_source_,
5494 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005495
asaperssonfab67072017-04-04 05:51:49 -07005496 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005497 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005498 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005499
mflodmancc3d4422017-08-03 08:27:51 -07005500 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005501 fake_encoder_.SetQualityScaling(true);
5502}
5503
Åsa Persson139f4dc2019-08-02 09:29:58 +02005504TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
5505 webrtc::test::ScopedFieldTrials field_trials(
5506 "WebRTC-Video-QualityScalerSettings/"
5507 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5508 // Reset encoder for field trials to take effect.
5509 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005510 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5511 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 09:29:58 +02005512 const int kWidth = 640;
5513 const int kHeight = 360;
5514
Henrik Boström381d1092020-05-12 18:49:07 +02005515 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005516 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005517 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5518 // Frame should not be dropped.
5519 WaitForEncodedFrame(1);
5520
Henrik Boström381d1092020-05-12 18:49:07 +02005521 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005522 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5523 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5524 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005525 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5526 // Frame should not be dropped.
5527 WaitForEncodedFrame(2);
5528
Henrik Boström381d1092020-05-12 18:49:07 +02005529 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005530 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5531 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5532 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005533 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5534 // Expect to drop this frame, the wait should time out.
5535 ExpectDroppedFrame();
5536
5537 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005538 EXPECT_TRUE_WAIT(
5539 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005540 video_stream_encoder_->Stop();
5541}
5542
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005543TEST_F(VideoStreamEncoderTest,
5544 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
5545 webrtc::test::ScopedFieldTrials field_trials(
5546 "WebRTC-Video-QualityScalerSettings/"
5547 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5548 fake_encoder_.SetQualityScaling(false);
5549 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005550 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5551 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005552 const int kWidth = 640;
5553 const int kHeight = 360;
5554
5555 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005556 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005557 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5558 // Frame should not be dropped.
5559 WaitForEncodedFrame(1);
5560
5561 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5562 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5563 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5564 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5565 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5566 // Frame should not be dropped.
5567 WaitForEncodedFrame(2);
5568
5569 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5570 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5571 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5572 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5573 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5574 // Not dropped since quality scaling is disabled.
5575 WaitForEncodedFrame(3);
5576
5577 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005578 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005579 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5580
5581 video_stream_encoder_->Stop();
5582}
5583
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005584TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005585 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005586 // Set simulcast.
5587 ResetEncoder("VP8", 3, 1, 1, false);
5588 fake_encoder_.SetQualityScaling(true);
5589 const int kWidth = 1280;
5590 const int kHeight = 720;
5591 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005592 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005593 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5594 // Frame should not be dropped.
5595 WaitForEncodedFrame(1);
5596
5597 // Trigger QVGA "singlecast"
5598 // Update the config.
5599 VideoEncoderConfig video_encoder_config;
5600 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5601 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005602 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005603 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005604 "VP8", /*max qp*/ 56, /*screencast*/ false,
5605 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005606 for (auto& layer : video_encoder_config.simulcast_layers) {
5607 layer.num_temporal_layers = 1;
5608 layer.max_framerate = kDefaultFramerate;
5609 }
Asa Persson606d3cb2021-10-04 10:07:11 +02005610 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005611 video_encoder_config.content_type =
5612 VideoEncoderConfig::ContentType::kRealtimeVideo;
5613
5614 video_encoder_config.simulcast_layers[0].active = true;
5615 video_encoder_config.simulcast_layers[1].active = false;
5616 video_encoder_config.simulcast_layers[2].active = false;
5617
5618 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5619 kMaxPayloadLength);
5620 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5621
5622 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5623 // Frame should not be dropped.
5624 WaitForEncodedFrame(2);
5625
5626 // Trigger HD "singlecast"
5627 video_encoder_config.simulcast_layers[0].active = false;
5628 video_encoder_config.simulcast_layers[1].active = false;
5629 video_encoder_config.simulcast_layers[2].active = true;
5630
5631 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5632 kMaxPayloadLength);
5633 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5634
5635 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5636 // Frame should be dropped because of initial frame drop.
5637 ExpectDroppedFrame();
5638
5639 // Expect the sink_wants to specify a scaled frame.
5640 EXPECT_TRUE_WAIT(
5641 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5642 video_stream_encoder_->Stop();
5643}
5644
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005645TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005646 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005647 // Set simulcast.
5648 ResetEncoder("VP9", 1, 1, 3, false);
5649 fake_encoder_.SetQualityScaling(true);
5650 const int kWidth = 1280;
5651 const int kHeight = 720;
5652 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005653 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005654 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5655 // Frame should not be dropped.
5656 WaitForEncodedFrame(1);
5657
5658 // Trigger QVGA "singlecast"
5659 // Update the config.
5660 VideoEncoderConfig video_encoder_config;
5661 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5662 &video_encoder_config);
5663 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5664 vp9_settings.numberOfSpatialLayers = 3;
5665 // Since only one layer is active - automatic resize should be enabled.
5666 vp9_settings.automaticResizeOn = true;
5667 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005668 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005669 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005670 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005671 video_encoder_config.content_type =
5672 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 16:22:11 +02005673 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005674 // which SVC layers are active.
5675 video_encoder_config.simulcast_layers.resize(3);
5676
5677 video_encoder_config.simulcast_layers[0].active = true;
5678 video_encoder_config.simulcast_layers[1].active = false;
5679 video_encoder_config.simulcast_layers[2].active = false;
5680
5681 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5682 kMaxPayloadLength);
5683 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5684
5685 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5686 // Frame should not be dropped.
5687 WaitForEncodedFrame(2);
5688
5689 // Trigger HD "singlecast"
5690 video_encoder_config.simulcast_layers[0].active = false;
5691 video_encoder_config.simulcast_layers[1].active = false;
5692 video_encoder_config.simulcast_layers[2].active = true;
5693
5694 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5695 kMaxPayloadLength);
5696 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5697
5698 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5699 // Frame should be dropped because of initial frame drop.
5700 ExpectDroppedFrame();
5701
5702 // Expect the sink_wants to specify a scaled frame.
5703 EXPECT_TRUE_WAIT(
5704 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5705 video_stream_encoder_->Stop();
5706}
5707
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005708TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005709 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5710 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5711 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5712 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5713 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5714 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5715 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5716 fake_encoder_.SetResolutionBitrateLimits(
5717 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5718
5719 VideoEncoderConfig video_encoder_config;
5720 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5721 &video_encoder_config);
5722 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5723 vp9_settings.numberOfSpatialLayers = 3;
5724 // Since only one layer is active - automatic resize should be enabled.
5725 vp9_settings.automaticResizeOn = true;
5726 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005727 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005728 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005729 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005730 video_encoder_config.content_type =
5731 VideoEncoderConfig::ContentType::kRealtimeVideo;
5732 // Simulcast layers are used to indicate which spatial layers are active.
5733 video_encoder_config.simulcast_layers.resize(3);
5734 video_encoder_config.simulcast_layers[0].active = false;
5735 video_encoder_config.simulcast_layers[1].active = true;
5736 video_encoder_config.simulcast_layers[2].active = false;
5737
5738 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5739 kMaxPayloadLength);
5740 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5741
5742 // The encoder bitrate limits for 360p should be used.
5743 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5744 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005745 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5746 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5747 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5748 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5749 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5750 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005751 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005752 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005753 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005754 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005755
5756 // The encoder bitrate limits for 270p should be used.
5757 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5758 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005759 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5760 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5761 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5762 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5763 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5764 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005765 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005766 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005767 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005768 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005769
5770 video_stream_encoder_->Stop();
5771}
5772
5773TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005774 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5775 VideoEncoderConfig video_encoder_config;
5776 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5777 &video_encoder_config);
5778 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5779 vp9_settings.numberOfSpatialLayers = 3;
5780 // Since only one layer is active - automatic resize should be enabled.
5781 vp9_settings.automaticResizeOn = true;
5782 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005783 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005784 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005785 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005786 video_encoder_config.content_type =
5787 VideoEncoderConfig::ContentType::kRealtimeVideo;
5788 // Simulcast layers are used to indicate which spatial layers are active.
5789 video_encoder_config.simulcast_layers.resize(3);
5790 video_encoder_config.simulcast_layers[0].active = false;
5791 video_encoder_config.simulcast_layers[1].active = true;
5792 video_encoder_config.simulcast_layers[2].active = false;
5793
5794 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5795 kMaxPayloadLength);
5796 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5797
5798 // The default bitrate limits for 360p should be used.
5799 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005800 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5801 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005802 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5803 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005804 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5805 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5806 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5807 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5808 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5809 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005810 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005811 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005812 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005813 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005814
5815 // The default bitrate limits for 270p should be used.
5816 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005817 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5818 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005819 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5820 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005821 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5822 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5823 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5824 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5825 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5826 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005827 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005828 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005829 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005830 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005831
5832 video_stream_encoder_->Stop();
5833}
5834
5835TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
5836 webrtc::test::ScopedFieldTrials field_trials(
5837 "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
5838 VideoEncoderConfig video_encoder_config;
5839 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5840 &video_encoder_config);
5841 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5842 vp9_settings.numberOfSpatialLayers = 3;
5843 // Since only one layer is active - automatic resize should be enabled.
5844 vp9_settings.automaticResizeOn = true;
5845 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005846 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005847 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005848 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005849 video_encoder_config.content_type =
5850 VideoEncoderConfig::ContentType::kRealtimeVideo;
5851 // Simulcast layers are used to indicate which spatial layers are active.
5852 video_encoder_config.simulcast_layers.resize(3);
5853 video_encoder_config.simulcast_layers[0].active = false;
5854 video_encoder_config.simulcast_layers[1].active = true;
5855 video_encoder_config.simulcast_layers[2].active = false;
5856
5857 // Reset encoder for field trials to take effect.
5858 ConfigureEncoder(video_encoder_config.Copy());
5859
5860 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5861 kMaxPayloadLength);
5862 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5863
5864 // The default bitrate limits for 360p should not be used.
5865 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005866 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5867 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005868 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5869 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005870 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5871 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5872 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5873 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5874 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5875 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005876 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005877 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005878
5879 video_stream_encoder_->Stop();
5880}
5881
5882TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5883 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5884 /*num_spatial_layers=*/1, /*screenshare=*/false);
5885
5886 // The default singlecast bitrate limits for 720p should not be used.
5887 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005888 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5889 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005890 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5891 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005892 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5893 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5894 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5895 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5896 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5897 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005898 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005899 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005900
5901 video_stream_encoder_->Stop();
5902}
5903
5904TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005905 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5906 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5907 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5908 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5909 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5910 fake_encoder_.SetResolutionBitrateLimits(
5911 {kEncoderLimits180p, kEncoderLimits720p});
5912
5913 VideoEncoderConfig video_encoder_config;
5914 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5915 &video_encoder_config);
5916 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5917 vp9_settings.numberOfSpatialLayers = 3;
5918 // Since only one layer is active - automatic resize should be enabled.
5919 vp9_settings.automaticResizeOn = true;
5920 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005921 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005922 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005923 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005924 video_encoder_config.content_type =
5925 VideoEncoderConfig::ContentType::kRealtimeVideo;
5926 // Simulcast layers are used to indicate which spatial layers are active.
5927 video_encoder_config.simulcast_layers.resize(3);
5928 video_encoder_config.simulcast_layers[0].active = true;
5929 video_encoder_config.simulcast_layers[1].active = false;
5930 video_encoder_config.simulcast_layers[2].active = false;
5931
5932 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5933 kMaxPayloadLength);
5934 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5935
5936 // Limits not applied on lowest stream, limits for 180p should not be used.
5937 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5938 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005939 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5940 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5941 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5942 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5943 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5944 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005945 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005946 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005947 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005948 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005949
5950 video_stream_encoder_->Stop();
5951}
5952
5953TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005954 InitialFrameDropActivatesWhenResolutionIncreases) {
5955 const int kWidth = 640;
5956 const int kHeight = 360;
5957
5958 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005959 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005960 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5961 // Frame should not be dropped.
5962 WaitForEncodedFrame(1);
5963
5964 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005965 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005966 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
5967 // Frame should not be dropped, bitrate not too low for frame.
5968 WaitForEncodedFrame(2);
5969
5970 // Incoming resolution increases.
5971 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5972 // Expect to drop this frame, bitrate too low for frame.
5973 ExpectDroppedFrame();
5974
5975 // Expect the sink_wants to specify a scaled frame.
5976 EXPECT_TRUE_WAIT(
5977 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5978 video_stream_encoder_->Stop();
5979}
5980
5981TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
5982 const int kWidth = 640;
5983 const int kHeight = 360;
5984 // So that quality scaling doesn't happen by itself.
5985 fake_encoder_.SetQp(kQpHigh);
5986
5987 AdaptingFrameForwarder source(&time_controller_);
5988 source.set_adaptation_enabled(true);
5989 video_stream_encoder_->SetSource(
5990 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
5991
5992 int timestamp = 1;
5993
5994 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005995 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005996 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
5997 WaitForEncodedFrame(timestamp);
5998 timestamp += 9000;
5999 // Long pause to disable all first BWE drop logic.
6000 AdvanceTime(TimeDelta::Millis(1000));
6001
6002 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006003 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006004 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6005 // Not dropped frame, as initial frame drop is disabled by now.
6006 WaitForEncodedFrame(timestamp);
6007 timestamp += 9000;
6008 AdvanceTime(TimeDelta::Millis(100));
6009
6010 // Quality adaptation down.
6011 video_stream_encoder_->TriggerQualityLow();
6012
6013 // Adaptation has an effect.
6014 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6015 5000);
6016
6017 // Frame isn't dropped as initial frame dropper is disabled.
6018 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6019 WaitForEncodedFrame(timestamp);
6020 timestamp += 9000;
6021 AdvanceTime(TimeDelta::Millis(100));
6022
6023 // Quality adaptation up.
6024 video_stream_encoder_->TriggerQualityHigh();
6025
6026 // Adaptation has an effect.
6027 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
6028 5000);
6029
6030 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6031 // Frame should not be dropped, as initial framedropper is off.
6032 WaitForEncodedFrame(timestamp);
6033
6034 video_stream_encoder_->Stop();
6035}
6036
Åsa Persson7f354f82021-02-04 15:52:15 +01006037TEST_F(VideoStreamEncoderTest,
6038 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
6039 const int kMinStartBps360p = 222000;
6040 fake_encoder_.SetResolutionBitrateLimits(
6041 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6042 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6043 800000)});
6044
6045 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6046 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6047 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6048 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
6049 0, 0, 0);
6050 // Frame should not be dropped, bitrate not too low for frame.
6051 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6052 WaitForEncodedFrame(1);
6053
6054 // Incoming resolution increases, initial frame drop activates.
6055 // Frame should be dropped, link allocation too low for frame.
6056 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6057 ExpectDroppedFrame();
6058
6059 // Expect sink_wants to specify a scaled frame.
6060 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6061 5000);
6062 video_stream_encoder_->Stop();
6063}
6064
6065TEST_F(VideoStreamEncoderTest,
6066 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6067 const int kMinStartBps360p = 222000;
6068 fake_encoder_.SetResolutionBitrateLimits(
6069 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6070 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6071 800000)});
6072
6073 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6074 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6075 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6076 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6077 0, 0, 0);
6078 // Frame should not be dropped, bitrate not too low for frame.
6079 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6080 WaitForEncodedFrame(1);
6081
6082 // Incoming resolution increases, initial frame drop activates.
6083 // Frame should be dropped, link allocation not too low for frame.
6084 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6085 WaitForEncodedFrame(2);
6086
6087 video_stream_encoder_->Stop();
6088}
6089
Åsa Perssone644a032019-11-08 15:56:00 +01006090TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
6091 webrtc::test::ScopedFieldTrials field_trials(
Åsa Persson06defc42021-09-10 15:28:48 +02006092 "WebRTC-Video-QualityRampupSettings/"
6093 "min_pixels:921600,min_duration_ms:2000/");
6094
6095 const int kWidth = 1280;
6096 const int kHeight = 720;
6097 const int kFps = 10;
6098 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006099
6100 // Reset encoder for field trials to take effect.
6101 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 10:07:11 +02006102 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 11:34:32 +02006103 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01006104 ConfigureEncoder(std::move(config));
6105 fake_encoder_.SetQp(kQpLow);
6106
6107 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006108 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01006109 source.set_adaptation_enabled(true);
6110 video_stream_encoder_->SetSource(&source,
6111 DegradationPreference::MAINTAIN_FRAMERATE);
6112
6113 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 10:07:11 +02006114 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 18:49:07 +02006115 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006116 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006117
6118 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 15:28:48 +02006119 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006120 int64_t timestamp_ms = kFrameIntervalMs;
6121 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6122 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02006123 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6124 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01006125
6126 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02006127 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6128 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006129
Artem Titovab30d722021-07-27 16:22:11 +02006130 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 15:28:48 +02006131 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 15:56:00 +01006132 for (size_t i = 1; i <= 10; i++) {
6133 timestamp_ms += kFrameIntervalMs;
6134 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6135 WaitForEncodedFrame(timestamp_ms);
6136 }
Åsa Persson06defc42021-09-10 15:28:48 +02006137
6138 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6139 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6140 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6141 timestamp_ms += kFrameIntervalMs;
6142 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6143 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 15:56:00 +01006144 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6145 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6146
Åsa Persson06defc42021-09-10 15:28:48 +02006147 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 15:56:00 +01006148 timestamp_ms += kFrameIntervalMs;
6149 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6150 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006151 // The ramp-up code involves the adaptation queue, give it time to execute.
6152 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006153 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006154 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006155
6156 // Frame should not be adapted.
6157 timestamp_ms += kFrameIntervalMs;
6158 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6159 WaitForEncodedFrame(kWidth, kHeight);
6160 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6161
6162 video_stream_encoder_->Stop();
6163}
6164
mflodmancc3d4422017-08-03 08:27:51 -07006165TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006166 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Ilya Nikolaevskiy483b31c2021-02-03 17:19:31 +01006167 webrtc::test::ScopedFieldTrials field_trials(
6168 "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006169 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006170 source.set_adaptation_enabled(true);
6171 video_stream_encoder_->SetSource(&source,
6172 DegradationPreference::MAINTAIN_FRAMERATE);
6173 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006174 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006175 fake_encoder_.SetQp(kQpHigh + 1);
6176 const int kWidth = 1280;
6177 const int kHeight = 720;
6178 const int64_t kFrameIntervalMs = 100;
6179 int64_t timestamp_ms = kFrameIntervalMs;
6180 for (size_t i = 1; i <= 100; i++) {
6181 timestamp_ms += kFrameIntervalMs;
6182 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6183 WaitForEncodedFrame(timestamp_ms);
6184 }
6185 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6186 // for the first time.
6187 // TODO(eshr): We should avoid these waits by using threads with simulated
6188 // time.
6189 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6190 2000 * 2.5 * 2);
6191 timestamp_ms += kFrameIntervalMs;
6192 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6193 WaitForEncodedFrame(timestamp_ms);
6194 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6195 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6196 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6197
6198 // Disable Quality scaling by turning off scaler on the encoder and
6199 // reconfiguring.
6200 fake_encoder_.SetQualityScaling(false);
6201 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6202 kMaxPayloadLength);
6203 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Markus Handell28c71802021-11-08 10:11:55 +01006204 AdvanceTime(TimeDelta::Zero());
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006205 // Since we turned off the quality scaler, the adaptations made by it are
6206 // removed.
6207 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6208 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6209
6210 video_stream_encoder_->Stop();
6211}
6212
6213TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006214 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6215 const int kTooSmallWidth = 10;
6216 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006217 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006218 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006219
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006220 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006221 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006222 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006223 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006224 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006225 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6226
6227 // Trigger adapt down, too small frame, expect no change.
6228 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006229 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006230 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006231 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006232 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6233 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6234
mflodmancc3d4422017-08-03 08:27:51 -07006235 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006236}
6237
mflodmancc3d4422017-08-03 08:27:51 -07006238TEST_F(VideoStreamEncoderTest,
6239 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006240 const int kTooSmallWidth = 10;
6241 const int kTooSmallHeight = 10;
6242 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006243 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006244 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006245
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006246 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006247 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006248 video_stream_encoder_->SetSource(&source,
6249 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006250 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006251 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6252 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6253
6254 // Trigger adapt down, expect limited framerate.
6255 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006256 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006257 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006258 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006259 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6260 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6261 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6262
6263 // Trigger adapt down, too small frame, expect no change.
6264 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006265 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006266 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006267 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006268 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6269 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6270 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6271
mflodmancc3d4422017-08-03 08:27:51 -07006272 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006273}
6274
mflodmancc3d4422017-08-03 08:27:51 -07006275TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006276 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006277 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006278 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006279 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006280 const int kFrameWidth = 1280;
6281 const int kFrameHeight = 720;
6282 video_source_.IncomingCapturedFrame(
6283 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006284 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006285 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006286}
6287
sprangb1ca0732017-02-01 08:38:12 -08006288// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006289TEST_F(VideoStreamEncoderTest,
6290 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006291 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006292 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006293
6294 const int kFrameWidth = 1280;
6295 const int kFrameHeight = 720;
6296 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006297 // requested by
6298 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006299 video_source_.set_adaptation_enabled(true);
6300
6301 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006302 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006303 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006304
6305 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006306 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006307 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006308 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006309 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006310
asaperssonfab67072017-04-04 05:51:49 -07006311 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006312 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006313 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006314 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006315 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006316
mflodmancc3d4422017-08-03 08:27:51 -07006317 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006318}
sprangfe627f32017-03-29 08:24:59 -07006319
mflodmancc3d4422017-08-03 08:27:51 -07006320TEST_F(VideoStreamEncoderTest,
6321 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006322 const int kFrameWidth = 1280;
6323 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006324
Henrik Boström381d1092020-05-12 18:49:07 +02006325 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006326 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006327 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006328 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006329 video_source_.set_adaptation_enabled(true);
6330
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006331 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006332
6333 video_source_.IncomingCapturedFrame(
6334 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006335 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006336
6337 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006338 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006339
6340 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006341 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006342 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006343 video_source_.IncomingCapturedFrame(
6344 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006345 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006346 }
6347
6348 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006349 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006350 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006351 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006352 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006353 video_source_.IncomingCapturedFrame(
6354 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006355 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006356 ++num_frames_dropped;
6357 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006358 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006359 }
6360 }
6361
sprang4847ae62017-06-27 07:06:52 -07006362 // Add some slack to account for frames dropped by the frame dropper.
6363 const int kErrorMargin = 1;
6364 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006365 kErrorMargin);
6366
6367 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006368 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006369 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006370 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006371 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006372 video_source_.IncomingCapturedFrame(
6373 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006374 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006375 ++num_frames_dropped;
6376 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006377 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006378 }
6379 }
sprang4847ae62017-06-27 07:06:52 -07006380 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006381 kErrorMargin);
6382
6383 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006384 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006385 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006386 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006387 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006388 video_source_.IncomingCapturedFrame(
6389 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006390 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006391 ++num_frames_dropped;
6392 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006393 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006394 }
6395 }
sprang4847ae62017-06-27 07:06:52 -07006396 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006397 kErrorMargin);
6398
6399 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006400 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006401 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006402 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006403 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006404 video_source_.IncomingCapturedFrame(
6405 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006406 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006407 ++num_frames_dropped;
6408 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006409 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006410 }
6411 }
6412 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6413
mflodmancc3d4422017-08-03 08:27:51 -07006414 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006415}
6416
mflodmancc3d4422017-08-03 08:27:51 -07006417TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006418 const int kFramerateFps = 5;
6419 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006420 const int kFrameWidth = 1280;
6421 const int kFrameHeight = 720;
6422
sprang4847ae62017-06-27 07:06:52 -07006423 // Reconfigure encoder with two temporal layers and screensharing, which will
6424 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006425 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006426
Henrik Boström381d1092020-05-12 18:49:07 +02006427 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006428 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006429 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006430 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006431 video_source_.set_adaptation_enabled(true);
6432
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006433 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006434
6435 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006436 rtc::VideoSinkWants last_wants;
6437 do {
6438 last_wants = video_source_.sink_wants();
6439
sprangc5d62e22017-04-02 23:53:04 -07006440 // Insert frames to get a new fps estimate...
6441 for (int j = 0; j < kFramerateFps; ++j) {
6442 video_source_.IncomingCapturedFrame(
6443 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006444 if (video_source_.last_sent_width()) {
6445 sink_.WaitForEncodedFrame(timestamp_ms);
6446 }
sprangc5d62e22017-04-02 23:53:04 -07006447 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006448 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006449 }
6450 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006451 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006452 } while (video_source_.sink_wants().max_framerate_fps <
6453 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006454
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006455 EXPECT_THAT(video_source_.sink_wants(),
6456 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006457
mflodmancc3d4422017-08-03 08:27:51 -07006458 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006459}
asaperssonf7e294d2017-06-13 23:25:22 -07006460
mflodmancc3d4422017-08-03 08:27:51 -07006461TEST_F(VideoStreamEncoderTest,
6462 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006463 const int kWidth = 1280;
6464 const int kHeight = 720;
6465 const int64_t kFrameIntervalMs = 150;
6466 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006467 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006468 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006469
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006470 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006471 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006472 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006473 video_stream_encoder_->SetSource(&source,
6474 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006475 timestamp_ms += kFrameIntervalMs;
6476 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006477 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006478 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006479 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6480 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6481 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6482
6483 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006484 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006485 timestamp_ms += kFrameIntervalMs;
6486 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006487 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006488 EXPECT_THAT(source.sink_wants(),
6489 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006490 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6491 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6492 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6493
6494 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006495 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006496 timestamp_ms += kFrameIntervalMs;
6497 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006498 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006499 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006500 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6501 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6502 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6503
6504 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006505 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006506 timestamp_ms += kFrameIntervalMs;
6507 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006508 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006509 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006510 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6511 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6512 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6513
6514 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006515 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006516 timestamp_ms += kFrameIntervalMs;
6517 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006518 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006519 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006520 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6521 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6522 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6523
6524 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006525 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006526 timestamp_ms += kFrameIntervalMs;
6527 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006528 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006529 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006530 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6531 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6532 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6533
6534 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006535 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006536 timestamp_ms += kFrameIntervalMs;
6537 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006538 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006539 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006540 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6541 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6542 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6543
6544 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006545 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006546 timestamp_ms += kFrameIntervalMs;
6547 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006548 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006549 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006550 rtc::VideoSinkWants last_wants = source.sink_wants();
6551 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6552 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6553 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6554
6555 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006556 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006557 timestamp_ms += kFrameIntervalMs;
6558 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006559 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006560 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006561 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6562 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6563 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6564
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006565 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006566 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006567 timestamp_ms += kFrameIntervalMs;
6568 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006569 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006570 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006571 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6572 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6573 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6574
6575 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006576 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006577 timestamp_ms += kFrameIntervalMs;
6578 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006579 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006580 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006581 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6582 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6583 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6584
6585 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006586 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006587 timestamp_ms += kFrameIntervalMs;
6588 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006589 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006590 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006591 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6592 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6593 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6594
6595 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006596 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006597 timestamp_ms += kFrameIntervalMs;
6598 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006599 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006600 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006601 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6602 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6603 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6604
6605 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006606 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006607 timestamp_ms += kFrameIntervalMs;
6608 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006609 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006610 EXPECT_THAT(source.sink_wants(), FpsMax());
6611 EXPECT_EQ(source.sink_wants().max_pixel_count,
6612 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006613 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6614 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6615 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6616
6617 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006618 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006619 timestamp_ms += kFrameIntervalMs;
6620 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006621 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006622 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006623 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6624 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6625 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6626
Åsa Persson30ab0152019-08-27 12:22:33 +02006627 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006628 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006629 timestamp_ms += kFrameIntervalMs;
6630 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006631 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006632 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006633 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006634 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6635 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6636 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6637
6638 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006639 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006640 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006641 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6642
mflodmancc3d4422017-08-03 08:27:51 -07006643 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006644}
6645
mflodmancc3d4422017-08-03 08:27:51 -07006646TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006647 const int kWidth = 1280;
6648 const int kHeight = 720;
6649 const int64_t kFrameIntervalMs = 150;
6650 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006651 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006652 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006653
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006654 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006655 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006656 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006657 video_stream_encoder_->SetSource(&source,
6658 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006659 timestamp_ms += kFrameIntervalMs;
6660 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006661 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006662 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006663 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6664 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6665 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6666 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6667 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6668 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6669
6670 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006671 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006672 timestamp_ms += kFrameIntervalMs;
6673 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006674 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006675 EXPECT_THAT(source.sink_wants(),
6676 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006677 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6678 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6679 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6680 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6681 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6682 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6683
6684 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006685 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006686 timestamp_ms += kFrameIntervalMs;
6687 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006688 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006689 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006690 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6691 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6692 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6693 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6694 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6695 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6696
6697 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006698 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006699 timestamp_ms += kFrameIntervalMs;
6700 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006701 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006702 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006703 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006704 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6705 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6706 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6707 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6708 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6709
Evan Shrubsole64469032020-06-11 10:45:29 +02006710 // Trigger cpu adapt up, expect no change since QP is most limited.
6711 {
6712 // Store current sink wants since we expect no change and if there is no
6713 // change then last_wants() is not updated.
6714 auto previous_sink_wants = source.sink_wants();
6715 video_stream_encoder_->TriggerCpuUnderuse();
6716 timestamp_ms += kFrameIntervalMs;
6717 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6718 WaitForEncodedFrame(timestamp_ms);
6719 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6720 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6721 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6722 }
6723
6724 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6725 video_stream_encoder_->TriggerQualityHigh();
6726 timestamp_ms += kFrameIntervalMs;
6727 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6728 WaitForEncodedFrame(timestamp_ms);
6729 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6730 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6731 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6732 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6733 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6734 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6735 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6736
6737 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6738 // expect increased resolution (960x540@30fps).
6739 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006740 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006741 timestamp_ms += kFrameIntervalMs;
6742 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006743 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006744 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006745 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6746 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6747 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6748 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6749 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006750 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006751
Evan Shrubsole64469032020-06-11 10:45:29 +02006752 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6753 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006754 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006755 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006756 timestamp_ms += kFrameIntervalMs;
6757 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006758 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006759 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006760 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006761 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6762 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6763 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6764 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6765 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006766 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006767
6768 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006769 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006770 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006771 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006772 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006773
mflodmancc3d4422017-08-03 08:27:51 -07006774 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006775}
6776
mflodmancc3d4422017-08-03 08:27:51 -07006777TEST_F(VideoStreamEncoderTest,
6778 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006779 const int kWidth = 640;
6780 const int kHeight = 360;
6781 const int kFpsLimit = 15;
6782 const int64_t kFrameIntervalMs = 150;
6783 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006784 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006785 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006786
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006787 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006788 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006789 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006790 video_stream_encoder_->SetSource(&source,
6791 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006792 timestamp_ms += kFrameIntervalMs;
6793 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006794 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006795 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006796 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6797 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6798 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6799 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6800 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6801 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6802
6803 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006804 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006805 timestamp_ms += kFrameIntervalMs;
6806 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006807 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006808 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006809 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6810 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6811 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6812 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6813 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6814 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6815
6816 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006817 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006818 timestamp_ms += kFrameIntervalMs;
6819 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006820 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006821 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006822 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006823 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006824 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6825 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6826 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6827 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6828
Evan Shrubsole64469032020-06-11 10:45:29 +02006829 // Trigger cpu adapt up, expect no change because quality is most limited.
6830 {
6831 auto previous_sink_wants = source.sink_wants();
6832 // Store current sink wants since we expect no change ind if there is no
6833 // change then last__wants() is not updated.
6834 video_stream_encoder_->TriggerCpuUnderuse();
6835 timestamp_ms += kFrameIntervalMs;
6836 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6837 WaitForEncodedFrame(timestamp_ms);
6838 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6839 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6840 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6841 }
6842
6843 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6844 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006845 timestamp_ms += kFrameIntervalMs;
6846 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006847 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006848 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006849 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6850 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6851 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006852 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6853 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6854 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006855
Evan Shrubsole64469032020-06-11 10:45:29 +02006856 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006857 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006858 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006859 timestamp_ms += kFrameIntervalMs;
6860 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006861 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006862 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006863 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6864 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6865 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6866 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6867 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006868 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006869
6870 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006871 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006872 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006873 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006874 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006875
mflodmancc3d4422017-08-03 08:27:51 -07006876 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006877}
6878
mflodmancc3d4422017-08-03 08:27:51 -07006879TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006880 const int kFrameWidth = 1920;
6881 const int kFrameHeight = 1080;
6882 // 3/4 of 1920.
6883 const int kAdaptedFrameWidth = 1440;
6884 // 3/4 of 1080 rounded down to multiple of 4.
6885 const int kAdaptedFrameHeight = 808;
6886 const int kFramerate = 24;
6887
Henrik Boström381d1092020-05-12 18:49:07 +02006888 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006889 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006890 // Trigger reconfigure encoder (without resetting the entire instance).
6891 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006892 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6893 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02006894 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 06:53:48 -07006895 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006896 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006897 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006898 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006899 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006900
6901 video_source_.set_adaptation_enabled(true);
6902
6903 video_source_.IncomingCapturedFrame(
6904 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006905 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006906
6907 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006908 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006909 video_source_.IncomingCapturedFrame(
6910 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006911 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006912
mflodmancc3d4422017-08-03 08:27:51 -07006913 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006914}
6915
mflodmancc3d4422017-08-03 08:27:51 -07006916TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006917 const int kFrameWidth = 1280;
6918 const int kFrameHeight = 720;
6919 const int kLowFps = 2;
6920 const int kHighFps = 30;
6921
Henrik Boström381d1092020-05-12 18:49:07 +02006922 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006923 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006924
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006925 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006926 max_framerate_ = kLowFps;
6927
6928 // Insert 2 seconds of 2fps video.
6929 for (int i = 0; i < kLowFps * 2; ++i) {
6930 video_source_.IncomingCapturedFrame(
6931 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6932 WaitForEncodedFrame(timestamp_ms);
6933 timestamp_ms += 1000 / kLowFps;
6934 }
6935
6936 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006937 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006938 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006939 video_source_.IncomingCapturedFrame(
6940 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6941 WaitForEncodedFrame(timestamp_ms);
6942 timestamp_ms += 1000 / kLowFps;
6943
6944 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6945
6946 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006947 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006948 const int kFrameIntervalMs = 1000 / kHighFps;
6949 max_framerate_ = kHighFps;
6950 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6951 video_source_.IncomingCapturedFrame(
6952 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6953 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6954 // be dropped if the encoder hans't been updated with the new higher target
6955 // framerate yet, causing it to overshoot the target bitrate and then
6956 // suffering the wrath of the media optimizer.
6957 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6958 timestamp_ms += kFrameIntervalMs;
6959 }
6960
6961 // Don expect correct measurement just yet, but it should be higher than
6962 // before.
6963 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
6964
mflodmancc3d4422017-08-03 08:27:51 -07006965 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07006966}
6967
mflodmancc3d4422017-08-03 08:27:51 -07006968TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07006969 const int kFrameWidth = 1280;
6970 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 15:09:05 +02006971 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01006972 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02006973 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07006974
Henrik Boström381d1092020-05-12 18:49:07 +02006975 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006976 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006977 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07006978
6979 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006980 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006981 video_source_.IncomingCapturedFrame(
6982 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6983 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02006984 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006985
6986 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02006987 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006988 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07006989
6990 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02006991 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006992 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07006993
Per Kjellanderdcef6412020-10-07 15:09:05 +02006994 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07006995 video_source_.IncomingCapturedFrame(
6996 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6997 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02006998 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07006999
mflodmancc3d4422017-08-03 08:27:51 -07007000 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007001}
ilnik6b826ef2017-06-16 06:53:48 -07007002
Niels Möller4db138e2018-04-19 09:04:13 +02007003TEST_F(VideoStreamEncoderTest,
7004 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
7005 const int kFrameWidth = 1280;
7006 const int kFrameHeight = 720;
7007 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02007008 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007009 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007010 video_source_.IncomingCapturedFrame(
7011 CreateFrame(1, kFrameWidth, kFrameHeight));
7012 WaitForEncodedFrame(1);
7013 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7014 .low_encode_usage_threshold_percent,
7015 default_options.low_encode_usage_threshold_percent);
7016 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7017 .high_encode_usage_threshold_percent,
7018 default_options.high_encode_usage_threshold_percent);
7019 video_stream_encoder_->Stop();
7020}
7021
7022TEST_F(VideoStreamEncoderTest,
7023 HigherCpuAdaptationThresholdsForHardwareEncoder) {
7024 const int kFrameWidth = 1280;
7025 const int kFrameHeight = 720;
7026 CpuOveruseOptions hardware_options;
7027 hardware_options.low_encode_usage_threshold_percent = 150;
7028 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01007029 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02007030
Henrik Boström381d1092020-05-12 18:49:07 +02007031 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007032 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007033 video_source_.IncomingCapturedFrame(
7034 CreateFrame(1, kFrameWidth, kFrameHeight));
7035 WaitForEncodedFrame(1);
7036 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7037 .low_encode_usage_threshold_percent,
7038 hardware_options.low_encode_usage_threshold_percent);
7039 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7040 .high_encode_usage_threshold_percent,
7041 hardware_options.high_encode_usage_threshold_percent);
7042 video_stream_encoder_->Stop();
7043}
7044
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007045TEST_F(VideoStreamEncoderTest,
7046 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
7047 const int kFrameWidth = 1280;
7048 const int kFrameHeight = 720;
7049
7050 const CpuOveruseOptions default_options;
7051 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007052 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007053 video_source_.IncomingCapturedFrame(
7054 CreateFrame(1, kFrameWidth, kFrameHeight));
7055 WaitForEncodedFrame(1);
7056 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7057 .low_encode_usage_threshold_percent,
7058 default_options.low_encode_usage_threshold_percent);
7059 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7060 .high_encode_usage_threshold_percent,
7061 default_options.high_encode_usage_threshold_percent);
7062
7063 CpuOveruseOptions hardware_options;
7064 hardware_options.low_encode_usage_threshold_percent = 150;
7065 hardware_options.high_encode_usage_threshold_percent = 200;
7066 fake_encoder_.SetIsHardwareAccelerated(true);
7067
7068 video_source_.IncomingCapturedFrame(
7069 CreateFrame(2, kFrameWidth, kFrameHeight));
7070 WaitForEncodedFrame(2);
7071
7072 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7073 .low_encode_usage_threshold_percent,
7074 hardware_options.low_encode_usage_threshold_percent);
7075 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7076 .high_encode_usage_threshold_percent,
7077 hardware_options.high_encode_usage_threshold_percent);
7078
7079 video_stream_encoder_->Stop();
7080}
7081
Niels Möller6bb5ab92019-01-11 11:11:10 +01007082TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7083 const int kFrameWidth = 320;
7084 const int kFrameHeight = 240;
7085 const int kFps = 30;
Asa Persson606d3cb2021-10-04 10:07:11 +02007086 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007087 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7088
Henrik Boström381d1092020-05-12 18:49:07 +02007089 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007090 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007091
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007092 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007093 max_framerate_ = kFps;
7094
7095 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7096 fake_encoder_.SimulateOvershoot(1.0);
7097 int num_dropped = 0;
7098 for (int i = 0; i < kNumFramesInRun; ++i) {
7099 video_source_.IncomingCapturedFrame(
7100 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7101 // Wait up to two frame durations for a frame to arrive.
7102 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7103 ++num_dropped;
7104 }
7105 timestamp_ms += 1000 / kFps;
7106 }
7107
Erik Språnga8d48ab2019-02-08 14:17:40 +01007108 // Framerate should be measured to be near the expected target rate.
7109 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7110
7111 // Frame drops should be within 5% of expected 0%.
7112 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007113
7114 // Make encoder produce frames at double the expected bitrate during 3 seconds
7115 // of video, verify number of drops. Rate needs to be slightly changed in
7116 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01007117 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02007118 const RateControlSettings trials =
7119 RateControlSettings::ParseFromFieldTrials();
7120 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01007121 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02007122 // frame dropping since the adjuter will try to just lower the target
7123 // bitrate rather than drop frames. If network headroom can be used, it
7124 // doesn't push back as hard so we don't need quite as much overshoot.
7125 // These numbers are unfortunately a bit magical but there's not trivial
7126 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01007127 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01007128 }
7129 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02007130 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007131 kTargetBitrate + DataRate::KilobitsPerSec(1),
7132 kTargetBitrate + DataRate::KilobitsPerSec(1),
7133 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007134 num_dropped = 0;
7135 for (int i = 0; i < kNumFramesInRun; ++i) {
7136 video_source_.IncomingCapturedFrame(
7137 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7138 // Wait up to two frame durations for a frame to arrive.
7139 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7140 ++num_dropped;
7141 }
7142 timestamp_ms += 1000 / kFps;
7143 }
7144
Henrik Boström381d1092020-05-12 18:49:07 +02007145 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007146 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007147
7148 // Target framerate should be still be near the expected target, despite
7149 // the frame drops.
7150 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7151
7152 // Frame drops should be within 5% of expected 50%.
7153 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007154
7155 video_stream_encoder_->Stop();
7156}
7157
7158TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7159 const int kFrameWidth = 320;
7160 const int kFrameHeight = 240;
7161 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 10:07:11 +02007162 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007163
7164 ASSERT_GT(max_framerate_, kActualInputFps);
7165
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007166 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007167 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007168 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007169 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007170
7171 // Insert 3 seconds of video, with an input fps lower than configured max.
7172 for (int i = 0; i < kActualInputFps * 3; ++i) {
7173 video_source_.IncomingCapturedFrame(
7174 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7175 // Wait up to two frame durations for a frame to arrive.
7176 WaitForEncodedFrame(timestamp_ms);
7177 timestamp_ms += 1000 / kActualInputFps;
7178 }
7179
7180 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7181
7182 video_stream_encoder_->Stop();
7183}
7184
Markus Handell9a478b52021-11-18 16:07:01 +01007185TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007186 VideoFrame::UpdateRect rect;
Markus Handell9a478b52021-11-18 16:07:01 +01007187 test::FrameForwarder source;
7188 video_stream_encoder_->SetSource(&source,
7189 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02007190 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007191 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007192
Markus Handell9a478b52021-11-18 16:07:01 +01007193 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(1, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007194 WaitForEncodedFrame(1);
7195 // On the very first frame full update should be forced.
7196 rect = fake_encoder_.GetLastUpdateRect();
7197 EXPECT_EQ(rect.offset_x, 0);
7198 EXPECT_EQ(rect.offset_y, 0);
7199 EXPECT_EQ(rect.height, codec_height_);
7200 EXPECT_EQ(rect.width, codec_width_);
Markus Handell9a478b52021-11-18 16:07:01 +01007201 // Frame with NTP timestamp 2 will be dropped due to outstanding frames
7202 // scheduled for processing during encoder queue processing of frame 2.
7203 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(2, nullptr, 1));
7204 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(3, nullptr, 10));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007205 WaitForEncodedFrame(3);
7206 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7207 rect = fake_encoder_.GetLastUpdateRect();
7208 EXPECT_EQ(rect.offset_x, 1);
7209 EXPECT_EQ(rect.offset_y, 0);
7210 EXPECT_EQ(rect.width, 10);
7211 EXPECT_EQ(rect.height, 1);
7212
Markus Handell9a478b52021-11-18 16:07:01 +01007213 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(4, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007214 WaitForEncodedFrame(4);
7215 // Previous frame was encoded, so no accumulation should happen.
7216 rect = fake_encoder_.GetLastUpdateRect();
7217 EXPECT_EQ(rect.offset_x, 0);
7218 EXPECT_EQ(rect.offset_y, 0);
7219 EXPECT_EQ(rect.width, 1);
7220 EXPECT_EQ(rect.height, 1);
7221
7222 video_stream_encoder_->Stop();
7223}
7224
Erik Språngd7329ca2019-02-21 21:19:53 +01007225TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007226 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007227 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007228
7229 // First frame is always keyframe.
7230 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7231 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007232 EXPECT_THAT(
7233 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007234 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007235
7236 // Insert delta frame.
7237 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7238 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007239 EXPECT_THAT(
7240 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007241 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007242
7243 // Request next frame be a key-frame.
7244 video_stream_encoder_->SendKeyFrame();
7245 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7246 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007247 EXPECT_THAT(
7248 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007249 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007250
7251 video_stream_encoder_->Stop();
7252}
7253
7254TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7255 // Setup simulcast with three streams.
7256 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007257 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007258 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7259 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007260 // Wait for all three layers before triggering event.
7261 sink_.SetNumExpectedLayers(3);
7262
7263 // First frame is always keyframe.
7264 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7265 WaitForEncodedFrame(1);
7266 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007267 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7268 VideoFrameType::kVideoFrameKey,
7269 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007270
7271 // Insert delta frame.
7272 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7273 WaitForEncodedFrame(2);
7274 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007275 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7276 VideoFrameType::kVideoFrameDelta,
7277 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007278
7279 // Request next frame be a key-frame.
7280 // Only first stream is configured to produce key-frame.
7281 video_stream_encoder_->SendKeyFrame();
7282 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7283 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007284
7285 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7286 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007287 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007288 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007289 VideoFrameType::kVideoFrameKey,
7290 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007291
7292 video_stream_encoder_->Stop();
7293}
7294
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007295TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007296 // SPS contains VUI with restrictions on the maximum number of reordered
7297 // pictures, there is no need to rewrite the bitstream to enable faster
7298 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007299 ResetEncoder("H264", 1, 1, 1, false);
7300
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007301 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007302 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007303 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007304
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007305 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 10:07:11 +02007306 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007307
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007308 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7309 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007310
7311 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007312 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007313
7314 video_stream_encoder_->Stop();
7315}
7316
7317TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007318 // SPS does not contain VUI, the bitstream is will be rewritten with added
7319 // VUI with restrictions on the maximum number of reordered pictures to
7320 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007321 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7322 0x00, 0x00, 0x03, 0x03, 0xF4,
7323 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007324 ResetEncoder("H264", 1, 1, 1, false);
7325
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007326 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007327 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007328 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007329
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007330 fake_encoder_.SetEncodedImageData(
7331 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007332
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007333 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7334 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007335
7336 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007337 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007338
7339 video_stream_encoder_->Stop();
7340}
7341
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007342TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7343 const int kFrameWidth = 1280;
7344 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 10:07:11 +02007345 const DataRate kTargetBitrate =
7346 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007347
Henrik Boström381d1092020-05-12 18:49:07 +02007348 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007349 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007350 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7351
7352 // Insert a first video frame. It should be dropped because of downscale in
7353 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007354 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007355 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7356 frame.set_rotation(kVideoRotation_270);
7357 video_source_.IncomingCapturedFrame(frame);
7358
7359 ExpectDroppedFrame();
7360
7361 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007362 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007363 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7364 frame.set_rotation(kVideoRotation_90);
7365 video_source_.IncomingCapturedFrame(frame);
7366
7367 WaitForEncodedFrame(timestamp_ms);
7368 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7369
7370 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007371 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007372 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7373 frame.set_rotation(kVideoRotation_180);
7374 video_source_.IncomingCapturedFrame(frame);
7375
7376 WaitForEncodedFrame(timestamp_ms);
7377 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7378
7379 video_stream_encoder_->Stop();
7380}
7381
Erik Språng5056af02019-09-02 15:53:11 +02007382TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7383 const int kFrameWidth = 320;
7384 const int kFrameHeight = 180;
7385
7386 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007387 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007388 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7389 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7390 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007391 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007392 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007393 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007394
7395 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007396 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007397 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7398 frame.set_rotation(kVideoRotation_270);
7399 video_source_.IncomingCapturedFrame(frame);
7400 WaitForEncodedFrame(timestamp_ms);
7401
7402 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 10:07:11 +02007403 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007404 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7405 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007406 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007407 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007408 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007409 /*link_allocation=*/target_rate,
7410 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007411 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007412 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007413 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7414
7415 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7416 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7417 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007418 DataRate allocation_sum =
7419 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007420 EXPECT_EQ(min_rate, allocation_sum);
7421 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7422
7423 video_stream_encoder_->Stop();
7424}
7425
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007426TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007427 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007428 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007429 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007430 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007431 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7432 WaitForEncodedFrame(1);
7433
7434 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7435 ASSERT_TRUE(prev_rate_settings.has_value());
7436 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7437 kDefaultFramerate);
7438
7439 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7440 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7441 timestamp_ms += 1000 / kDefaultFramerate;
7442 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7443 WaitForEncodedFrame(timestamp_ms);
7444 }
7445 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7446 kDefaultFramerate);
7447 // Capture larger frame to trigger a reconfigure.
7448 codec_height_ *= 2;
7449 codec_width_ *= 2;
7450 timestamp_ms += 1000 / kDefaultFramerate;
7451 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7452 WaitForEncodedFrame(timestamp_ms);
7453
7454 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7455 auto current_rate_settings =
7456 fake_encoder_.GetAndResetLastRateControlSettings();
7457 // Ensure we have actually reconfigured twice
7458 // The rate settings should have been set again even though
7459 // they haven't changed.
7460 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007461 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007462
7463 video_stream_encoder_->Stop();
7464}
7465
philipeld9cc8c02019-09-16 14:53:40 +02007466struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007467 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007468 MOCK_METHOD(void,
7469 RequestEncoderSwitch,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007470 (const webrtc::SdpVideoFormat& format,
7471 bool allow_default_fallback),
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007472 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007473};
7474
philipel9b058032020-02-10 11:30:00 +01007475TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7476 constexpr int kDontCare = 100;
7477 StrictMock<MockEncoderSelector> encoder_selector;
7478 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7479 &fake_encoder_, &encoder_selector);
7480 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7481
7482 // Reset encoder for new configuration to take effect.
7483 ConfigureEncoder(video_encoder_config_.Copy());
7484
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007485 EXPECT_CALL(encoder_selector, OnCurrentEncoder);
philipel9b058032020-02-10 11:30:00 +01007486
7487 video_source_.IncomingCapturedFrame(
7488 CreateFrame(kDontCare, kDontCare, kDontCare));
Markus Handell28c71802021-11-08 10:11:55 +01007489 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007490 video_stream_encoder_->Stop();
7491
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007492 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007493 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007494 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7495 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007496 video_stream_encoder_.reset();
7497}
7498
7499TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7500 constexpr int kDontCare = 100;
7501
7502 NiceMock<MockEncoderSelector> encoder_selector;
7503 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7504 video_send_config_.encoder_settings.encoder_switch_request_callback =
7505 &switch_callback;
7506 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7507 &fake_encoder_, &encoder_selector);
7508 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7509
7510 // Reset encoder for new configuration to take effect.
7511 ConfigureEncoder(video_encoder_config_.Copy());
7512
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007513 ON_CALL(encoder_selector, OnAvailableBitrate)
philipel9b058032020-02-10 11:30:00 +01007514 .WillByDefault(Return(SdpVideoFormat("AV1")));
7515 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007516 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7517 /*allow_default_fallback=*/false));
philipel9b058032020-02-10 11:30:00 +01007518
Henrik Boström381d1092020-05-12 18:49:07 +02007519 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007520 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7521 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7522 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007523 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007524 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007525 /*cwnd_reduce_ratio=*/0);
Markus Handell28c71802021-11-08 10:11:55 +01007526 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007527
7528 video_stream_encoder_->Stop();
7529}
7530
7531TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7532 constexpr int kSufficientBitrateToNotDrop = 1000;
7533 constexpr int kDontCare = 100;
7534
7535 NiceMock<MockVideoEncoder> video_encoder;
7536 NiceMock<MockEncoderSelector> encoder_selector;
7537 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7538 video_send_config_.encoder_settings.encoder_switch_request_callback =
7539 &switch_callback;
7540 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7541 &video_encoder, &encoder_selector);
7542 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7543
7544 // Reset encoder for new configuration to take effect.
7545 ConfigureEncoder(video_encoder_config_.Copy());
7546
7547 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7548 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7549 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007550 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007551 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7552 /*stable_target_bitrate=*/
7553 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7554 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007555 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007556 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007557 /*cwnd_reduce_ratio=*/0);
7558
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007559 ON_CALL(video_encoder, Encode)
philipel9b058032020-02-10 11:30:00 +01007560 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007561 ON_CALL(encoder_selector, OnEncoderBroken)
philipel9b058032020-02-10 11:30:00 +01007562 .WillByDefault(Return(SdpVideoFormat("AV2")));
7563
7564 rtc::Event encode_attempted;
7565 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007566 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7567 /*allow_default_fallback=*/true))
7568 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
philipel9b058032020-02-10 11:30:00 +01007569
7570 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7571 encode_attempted.Wait(3000);
7572
Markus Handell28c71802021-11-08 10:11:55 +01007573 AdvanceTime(TimeDelta::Zero());
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007574
philipel9b058032020-02-10 11:30:00 +01007575 video_stream_encoder_->Stop();
7576
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007577 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7578 // to it's factory, so in order for the encoder instance in the
7579 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7580 // reset the `video_stream_encoder_` here.
7581 video_stream_encoder_.reset();
7582}
7583
7584TEST_F(VideoStreamEncoderTest, SwitchEncoderOnInitFailureWithEncoderSelector) {
7585 NiceMock<MockVideoEncoder> video_encoder;
7586 NiceMock<MockEncoderSelector> encoder_selector;
7587 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7588 video_send_config_.encoder_settings.encoder_switch_request_callback =
7589 &switch_callback;
7590 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7591 &video_encoder, &encoder_selector);
7592 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7593
7594 // Reset encoder for new configuration to take effect.
7595 ConfigureEncoder(video_encoder_config_.Copy());
7596
7597 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7598 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7599 /*round_trip_time_ms=*/0,
7600 /*cwnd_reduce_ratio=*/0);
7601 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7602
7603 ON_CALL(video_encoder, InitEncode(_, _))
7604 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7605 ON_CALL(encoder_selector, OnEncoderBroken)
7606 .WillByDefault(Return(SdpVideoFormat("AV2")));
7607
7608 rtc::Event encode_attempted;
7609 EXPECT_CALL(switch_callback,
7610 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7611 /*allow_default_fallback=*/true))
7612 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7613
7614 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7615 encode_attempted.Wait(3000);
7616
7617 AdvanceTime(TimeDelta::Zero());
7618
7619 video_stream_encoder_->Stop();
7620
7621 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7622 // to it's factory, so in order for the encoder instance in the
7623 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7624 // reset the `video_stream_encoder_` here.
7625 video_stream_encoder_.reset();
7626}
7627
7628TEST_F(VideoStreamEncoderTest,
7629 SwitchEncoderOnInitFailureWithoutEncoderSelector) {
7630 NiceMock<MockVideoEncoder> video_encoder;
7631 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7632 video_send_config_.encoder_settings.encoder_switch_request_callback =
7633 &switch_callback;
7634 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7635 &video_encoder, /*encoder_selector=*/nullptr);
7636 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7637
7638 // Reset encoder for new configuration to take effect.
7639 ConfigureEncoder(video_encoder_config_.Copy());
7640
7641 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7642 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7643 /*round_trip_time_ms=*/0,
7644 /*cwnd_reduce_ratio=*/0);
7645 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7646
7647 ON_CALL(video_encoder, InitEncode(_, _))
7648 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7649
7650 rtc::Event encode_attempted;
7651 EXPECT_CALL(switch_callback,
7652 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "VP8"),
7653 /*allow_default_fallback=*/true))
7654 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7655
7656 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7657 encode_attempted.Wait(3000);
7658
7659 AdvanceTime(TimeDelta::Zero());
7660
7661 video_stream_encoder_->Stop();
7662
7663 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007664 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007665 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7666 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007667 video_stream_encoder_.reset();
7668}
7669
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007670TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007671 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007672 const int kFrameWidth = 320;
7673 const int kFrameHeight = 180;
7674
7675 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007676 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007677 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007678 /*target_bitrate=*/rate,
7679 /*stable_target_bitrate=*/rate,
7680 /*link_allocation=*/rate,
7681 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007682 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007683 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007684
7685 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007686 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007687 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7688 frame.set_rotation(kVideoRotation_270);
7689 video_source_.IncomingCapturedFrame(frame);
7690 WaitForEncodedFrame(timestamp_ms);
7691 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7692
7693 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007694 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007695 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007696 /*target_bitrate=*/new_stable_rate,
7697 /*stable_target_bitrate=*/new_stable_rate,
7698 /*link_allocation=*/rate,
7699 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007700 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007701 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007702 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7703 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7704 video_stream_encoder_->Stop();
7705}
7706
7707TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007708 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007709 const int kFrameWidth = 320;
7710 const int kFrameHeight = 180;
7711
7712 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007713 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007714 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007715 /*target_bitrate=*/rate,
7716 /*stable_target_bitrate=*/rate,
7717 /*link_allocation=*/rate,
7718 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007719 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007720 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007721
7722 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007723 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007724 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7725 frame.set_rotation(kVideoRotation_270);
7726 video_source_.IncomingCapturedFrame(frame);
7727 WaitForEncodedFrame(timestamp_ms);
7728 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7729
7730 // Set a higher target rate without changing the link_allocation. Should not
7731 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007732 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007733 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007734 /*target_bitrate=*/rate,
7735 /*stable_target_bitrate=*/new_stable_rate,
7736 /*link_allocation=*/rate,
7737 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007738 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007739 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007740 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7741 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7742 video_stream_encoder_->Stop();
7743}
7744
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007745TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
7746 test::ScopedFieldTrials field_trials(
7747 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7748 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7749 const int kFramerateFps = 30;
7750 const int kWidth = 1920;
7751 const int kHeight = 1080;
7752 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7753 // Works on screenshare mode.
7754 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7755 // We rely on the automatic resolution adaptation, but we handle framerate
7756 // adaptation manually by mocking the stats proxy.
7757 video_source_.set_adaptation_enabled(true);
7758
7759 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007760 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007761 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007762 video_stream_encoder_->SetSource(&video_source_,
7763 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007764 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007765
7766 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7767 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7768
7769 // Pass enough frames with the full update to trigger animation detection.
7770 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007771 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007772 frame.set_ntp_time_ms(timestamp_ms);
7773 frame.set_timestamp_us(timestamp_ms * 1000);
7774 video_source_.IncomingCapturedFrame(frame);
7775 WaitForEncodedFrame(timestamp_ms);
7776 }
7777
7778 // Resolution should be limited.
7779 rtc::VideoSinkWants expected;
7780 expected.max_framerate_fps = kFramerateFps;
7781 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007782 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007783
7784 // Pass one frame with no known update.
7785 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007786 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007787 frame.set_ntp_time_ms(timestamp_ms);
7788 frame.set_timestamp_us(timestamp_ms * 1000);
7789 frame.clear_update_rect();
7790
7791 video_source_.IncomingCapturedFrame(frame);
7792 WaitForEncodedFrame(timestamp_ms);
7793
7794 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007795 EXPECT_THAT(video_source_.sink_wants(),
7796 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007797
7798 video_stream_encoder_->Stop();
7799}
7800
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007801TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7802 const int kWidth = 720; // 540p adapted down.
7803 const int kHeight = 405;
7804 const int kNumFrames = 3;
7805 // Works on screenshare mode.
7806 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7807 /*num_spatial_layers=*/2, /*screenshare=*/true);
7808
7809 video_source_.set_adaptation_enabled(true);
7810
7811 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007812 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007813
7814 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7815
7816 // Pass enough frames with the full update to trigger animation detection.
7817 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007818 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007819 frame.set_ntp_time_ms(timestamp_ms);
7820 frame.set_timestamp_us(timestamp_ms * 1000);
7821 video_source_.IncomingCapturedFrame(frame);
7822 WaitForEncodedFrame(timestamp_ms);
7823 }
7824
7825 video_stream_encoder_->Stop();
7826}
7827
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007828TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7829 const float downscale_factors[] = {4.0, 2.0, 1.0};
7830 const int number_layers =
7831 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7832 VideoEncoderConfig config;
7833 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7834 for (int i = 0; i < number_layers; ++i) {
7835 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7836 config.simulcast_layers[i].active = true;
7837 }
7838 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007839 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007840 "VP8", /*max qp*/ 56, /*screencast*/ false,
7841 /*screenshare enabled*/ false);
7842 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007843 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7844 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007845
7846 // First initialization.
7847 // Encoder should be initialized. Next frame should be key frame.
7848 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7849 sink_.SetNumExpectedLayers(number_layers);
7850 int64_t timestamp_ms = kFrameIntervalMs;
7851 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7852 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007853 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007854 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7855 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7856 VideoFrameType::kVideoFrameKey,
7857 VideoFrameType::kVideoFrameKey}));
7858
7859 // Disable top layer.
7860 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7861 config.simulcast_layers[number_layers - 1].active = false;
7862 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7863 sink_.SetNumExpectedLayers(number_layers - 1);
7864 timestamp_ms += kFrameIntervalMs;
7865 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7866 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007867 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007868 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7869 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7870 VideoFrameType::kVideoFrameDelta,
7871 VideoFrameType::kVideoFrameDelta}));
7872
7873 // Re-enable top layer.
7874 // Encoder should be re-initialized. Next frame should be key frame.
7875 config.simulcast_layers[number_layers - 1].active = true;
7876 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7877 sink_.SetNumExpectedLayers(number_layers);
7878 timestamp_ms += kFrameIntervalMs;
7879 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7880 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007881 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007882 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7883 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7884 VideoFrameType::kVideoFrameKey,
7885 VideoFrameType::kVideoFrameKey}));
7886
7887 // Top layer max rate change.
7888 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7889 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
7890 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7891 sink_.SetNumExpectedLayers(number_layers);
7892 timestamp_ms += kFrameIntervalMs;
7893 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7894 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007895 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007896 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7897 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7898 VideoFrameType::kVideoFrameDelta,
7899 VideoFrameType::kVideoFrameDelta}));
7900
7901 // Top layer resolution change.
7902 // Encoder should be re-initialized. Next frame should be key frame.
7903 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
7904 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7905 sink_.SetNumExpectedLayers(number_layers);
7906 timestamp_ms += kFrameIntervalMs;
7907 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7908 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007909 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007910 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7911 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7912 VideoFrameType::kVideoFrameKey,
7913 VideoFrameType::kVideoFrameKey}));
7914 video_stream_encoder_->Stop();
7915}
7916
Henrik Boström1124ed12021-02-25 10:30:39 +01007917TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
7918 const int kFrameWidth = 1280;
7919 const int kFrameHeight = 720;
7920
7921 SetUp();
7922 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007923 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01007924
7925 // Capturing a frame should reconfigure the encoder and expose the encoder
7926 // resolution, which is the same as the input frame.
7927 int64_t timestamp_ms = kFrameIntervalMs;
7928 video_source_.IncomingCapturedFrame(
7929 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7930 WaitForEncodedFrame(timestamp_ms);
7931 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7932 EXPECT_THAT(video_source_.sink_wants().resolutions,
7933 ::testing::ElementsAreArray(
7934 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
7935
7936 video_stream_encoder_->Stop();
7937}
7938
7939TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
7940 // Pick downscale factors such that we never encode at full resolution - this
7941 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 16:22:11 +02007942 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 10:30:39 +01007943 // encoder should not ask for the frame resolution. This allows video frames
7944 // to have the appearence of one resolution but optimize its internal buffers
7945 // for what is actually encoded.
7946 const size_t kNumSimulcastLayers = 3u;
7947 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
7948 const int kFrameWidth = 1280;
7949 const int kFrameHeight = 720;
7950 const rtc::VideoSinkWants::FrameSize kLayer0Size(
7951 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
7952 const rtc::VideoSinkWants::FrameSize kLayer1Size(
7953 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
7954 const rtc::VideoSinkWants::FrameSize kLayer2Size(
7955 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
7956
7957 VideoEncoderConfig config;
7958 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
7959 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
7960 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
7961 config.simulcast_layers[i].active = true;
7962 }
7963 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007964 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01007965 "VP8", /*max qp*/ 56, /*screencast*/ false,
7966 /*screenshare enabled*/ false);
7967 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007968 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7969 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01007970
7971 // Capture a frame with all layers active.
7972 int64_t timestamp_ms = kFrameIntervalMs;
7973 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
7974 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7975 video_source_.IncomingCapturedFrame(
7976 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7977 WaitForEncodedFrame(timestamp_ms);
7978 // Expect encoded resolutions to match the expected simulcast layers.
7979 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7980 EXPECT_THAT(
7981 video_source_.sink_wants().resolutions,
7982 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
7983
7984 // Capture a frame with one of the layers inactive.
7985 timestamp_ms += kFrameIntervalMs;
7986 config.simulcast_layers[2].active = false;
7987 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
7988 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7989 video_source_.IncomingCapturedFrame(
7990 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7991 WaitForEncodedFrame(timestamp_ms);
7992
7993 // Expect encoded resolutions to match the expected simulcast layers.
7994 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7995 EXPECT_THAT(video_source_.sink_wants().resolutions,
7996 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
7997
7998 // Capture a frame with all but one layer turned off.
7999 timestamp_ms += kFrameIntervalMs;
8000 config.simulcast_layers[1].active = false;
8001 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
8002 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8003 video_source_.IncomingCapturedFrame(
8004 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8005 WaitForEncodedFrame(timestamp_ms);
8006
8007 // Expect encoded resolutions to match the expected simulcast layers.
8008 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8009 EXPECT_THAT(video_source_.sink_wants().resolutions,
8010 ::testing::ElementsAreArray({kLayer0Size}));
8011
8012 video_stream_encoder_->Stop();
8013}
8014
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008015TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008016 ResetEncoder("VP8", 1, 1, 1, false);
8017
Niels Möller8b692902021-06-14 12:04:57 +02008018 // Force encoder reconfig.
8019 video_source_.IncomingCapturedFrame(
8020 CreateFrame(1, codec_width_, codec_height_));
8021 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8022
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008023 // Set QP on encoded frame and pass the frame to encode complete callback.
8024 // Since QP is present QP parsing won't be triggered and the original value
8025 // should be kept.
8026 EncodedImage encoded_image;
8027 encoded_image.qp_ = 123;
8028 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8029 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8030 CodecSpecificInfo codec_info;
8031 codec_info.codecType = kVideoCodecVP8;
8032 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8033 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8034 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
8035 video_stream_encoder_->Stop();
8036}
8037
8038TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008039 ResetEncoder("VP8", 1, 1, 1, false);
8040
Niels Möller8b692902021-06-14 12:04:57 +02008041 // Force encoder reconfig.
8042 video_source_.IncomingCapturedFrame(
8043 CreateFrame(1, codec_width_, codec_height_));
8044 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8045
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008046 // Pass an encoded frame without QP to encode complete callback. QP should be
8047 // parsed and set.
8048 EncodedImage encoded_image;
8049 encoded_image.qp_ = -1;
8050 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8051 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8052 CodecSpecificInfo codec_info;
8053 codec_info.codecType = kVideoCodecVP8;
8054 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8055 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8056 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
8057 video_stream_encoder_->Stop();
8058}
8059
8060TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
8061 webrtc::test::ScopedFieldTrials field_trials(
8062 "WebRTC-QpParsingKillSwitch/Enabled/");
8063
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008064 ResetEncoder("VP8", 1, 1, 1, false);
8065
Niels Möller8b692902021-06-14 12:04:57 +02008066 // Force encoder reconfig.
8067 video_source_.IncomingCapturedFrame(
8068 CreateFrame(1, codec_width_, codec_height_));
8069 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8070
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008071 EncodedImage encoded_image;
8072 encoded_image.qp_ = -1;
8073 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8074 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8075 CodecSpecificInfo codec_info;
8076 codec_info.codecType = kVideoCodecVP8;
8077 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8078 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8079 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
8080 video_stream_encoder_->Stop();
8081}
8082
Sergey Silkind19e3b92021-03-16 10:05:30 +00008083TEST_F(VideoStreamEncoderTest,
8084 QualityScalingNotAllowed_QualityScalingDisabled) {
8085 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8086
8087 // Disable scaling settings in encoder info.
8088 fake_encoder_.SetQualityScaling(false);
8089 // Disable quality scaling in encoder config.
8090 video_encoder_config.is_quality_scaling_allowed = false;
8091 ConfigureEncoder(std::move(video_encoder_config));
8092
8093 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008094 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008095
8096 test::FrameForwarder source;
8097 video_stream_encoder_->SetSource(
8098 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8099 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8100 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8101
8102 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8103 WaitForEncodedFrame(1);
8104 video_stream_encoder_->TriggerQualityLow();
8105 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8106
8107 video_stream_encoder_->Stop();
8108}
8109
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008110TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
8111 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8112
8113 // Disable scaling settings in encoder info.
8114 fake_encoder_.SetQualityScaling(false);
8115 // Set QP trusted in encoder info.
8116 fake_encoder_.SetIsQpTrusted(true);
8117 // Enable quality scaling in encoder config.
8118 video_encoder_config.is_quality_scaling_allowed = false;
8119 ConfigureEncoder(std::move(video_encoder_config));
8120
8121 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008122 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008123
8124 test::FrameForwarder source;
8125 video_stream_encoder_->SetSource(
8126 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8127 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8128 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8129
8130 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8131 WaitForEncodedFrame(1);
8132 video_stream_encoder_->TriggerQualityLow();
8133 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8134
8135 video_stream_encoder_->Stop();
8136}
8137
Shuhai Pengf2707702021-09-29 17:19:44 +08008138TEST_F(VideoStreamEncoderTest,
8139 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8140 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8141
8142 // Disable scaling settings in encoder info.
8143 fake_encoder_.SetQualityScaling(false);
8144 // Set QP trusted in encoder info.
8145 fake_encoder_.SetIsQpTrusted(true);
8146 // Enable quality scaling in encoder config.
8147 video_encoder_config.is_quality_scaling_allowed = false;
8148 ConfigureEncoder(std::move(video_encoder_config));
8149
8150 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008151 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008152
8153 test::FrameForwarder source;
8154 video_stream_encoder_->SetSource(
8155 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8156 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8157 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8158
8159 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8160 WaitForEncodedFrame(1);
8161 video_stream_encoder_->TriggerQualityLow();
8162 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8163
8164 video_stream_encoder_->Stop();
8165}
8166
8167TEST_F(VideoStreamEncoderTest,
8168 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8169 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8170
8171 // Disable scaling settings in encoder info.
8172 fake_encoder_.SetQualityScaling(false);
8173 // Set QP trusted in encoder info.
8174 fake_encoder_.SetIsQpTrusted(false);
8175 // Enable quality scaling in encoder config.
8176 video_encoder_config.is_quality_scaling_allowed = false;
8177 ConfigureEncoder(std::move(video_encoder_config));
8178
8179 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008180 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008181
8182 test::FrameForwarder source;
8183 video_stream_encoder_->SetSource(
8184 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8185 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8186 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8187
8188 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8189 WaitForEncodedFrame(1);
8190 video_stream_encoder_->TriggerQualityLow();
8191 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8192
8193 video_stream_encoder_->Stop();
8194}
8195
8196TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8197 // Set QP trusted in encoder info.
8198 fake_encoder_.SetIsQpTrusted(false);
8199
8200 const int MinEncBitrateKbps = 30;
8201 const int MaxEncBitrateKbps = 100;
8202 const int MinStartBitrateKbp = 50;
8203 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8204 /*frame_size_pixels=*/codec_width_ * codec_height_,
8205 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8206 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8207 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8208
8209 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008210 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008211
8212 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8213
8214 VideoEncoderConfig video_encoder_config;
8215 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8216 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8217 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8218 MinEncBitrateKbps * 1000;
8219 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8220 kMaxPayloadLength);
8221
8222 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8223 WaitForEncodedFrame(1);
8224 EXPECT_EQ(
8225 MaxEncBitrateKbps,
8226 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8227 EXPECT_EQ(
8228 MinEncBitrateKbps,
8229 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8230
8231 video_stream_encoder_->Stop();
8232}
8233
8234TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8235 // Set QP trusted in encoder info.
8236 fake_encoder_.SetIsQpTrusted(false);
8237
8238 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8239 EncoderInfoSettings::
8240 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8241 codec_width_ * codec_height_,
8242 EncoderInfoSettings::
8243 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8244 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8245
8246 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8247 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8248 const int TargetEncBitrate = MaxEncBitrate;
8249 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8250 DataRate::BitsPerSec(TargetEncBitrate),
8251 DataRate::BitsPerSec(TargetEncBitrate),
8252 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8253
8254 VideoEncoderConfig video_encoder_config;
8255 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8256 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8257 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8258 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8259 kMaxPayloadLength);
8260
8261 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8262 WaitForEncodedFrame(1);
8263 EXPECT_EQ(
8264 MaxEncBitrate / 1000,
8265 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8266 EXPECT_EQ(
8267 MinEncBitrate / 1000,
8268 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8269
8270 video_stream_encoder_->Stop();
8271}
8272
Sergey Silkind19e3b92021-03-16 10:05:30 +00008273#if !defined(WEBRTC_IOS)
8274// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8275// disabled by default on iOS.
8276TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8277 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8278
8279 // Disable scaling settings in encoder info.
8280 fake_encoder_.SetQualityScaling(false);
8281 // Enable quality scaling in encoder config.
8282 video_encoder_config.is_quality_scaling_allowed = true;
8283 ConfigureEncoder(std::move(video_encoder_config));
8284
8285 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008286 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008287
8288 test::FrameForwarder source;
8289 video_stream_encoder_->SetSource(
8290 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8291 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8292 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8293
8294 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8295 WaitForEncodedFrame(1);
8296 video_stream_encoder_->TriggerQualityLow();
8297 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8298
8299 video_stream_encoder_->Stop();
8300}
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008301
8302TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8303 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8304
8305 // Disable scaling settings in encoder info.
8306 fake_encoder_.SetQualityScaling(false);
8307 // Set QP trusted in encoder info.
8308 fake_encoder_.SetIsQpTrusted(true);
8309 // Enable quality scaling in encoder config.
8310 video_encoder_config.is_quality_scaling_allowed = true;
8311 ConfigureEncoder(std::move(video_encoder_config));
8312
8313 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008314 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008315
8316 test::FrameForwarder source;
8317 video_stream_encoder_->SetSource(
8318 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8319 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8320 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8321
8322 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8323 WaitForEncodedFrame(1);
8324 video_stream_encoder_->TriggerQualityLow();
8325 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8326
8327 video_stream_encoder_->Stop();
8328}
Shuhai Pengf2707702021-09-29 17:19:44 +08008329
8330TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8331 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8332
8333 // Disable scaling settings in encoder info.
8334 fake_encoder_.SetQualityScaling(false);
8335 // Set QP not trusted in encoder info.
8336 fake_encoder_.SetIsQpTrusted(false);
8337 // Enable quality scaling in encoder config.
8338 video_encoder_config.is_quality_scaling_allowed = true;
8339 ConfigureEncoder(std::move(video_encoder_config));
8340
8341 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008342 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008343
8344 test::FrameForwarder source;
8345 video_stream_encoder_->SetSource(
8346 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8347 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8348 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8349
8350 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8351 WaitForEncodedFrame(1);
8352 video_stream_encoder_->TriggerQualityLow();
8353 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8354 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8355 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8356
8357 video_stream_encoder_->Stop();
8358}
8359
8360TEST_F(VideoStreamEncoderTest,
8361 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8362 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8363
8364 // Disable scaling settings in encoder info.
8365 fake_encoder_.SetQualityScaling(false);
8366 // Set QP trusted in encoder info.
8367 fake_encoder_.SetIsQpTrusted(true);
8368 // Enable quality scaling in encoder config.
8369 video_encoder_config.is_quality_scaling_allowed = true;
8370 ConfigureEncoder(std::move(video_encoder_config));
8371
8372 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008373 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008374
8375 test::FrameForwarder source;
8376 video_stream_encoder_->SetSource(
8377 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8378 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8379 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8380
8381 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8382 WaitForEncodedFrame(1);
8383 video_stream_encoder_->TriggerQualityLow();
8384 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8385 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8386
8387 video_stream_encoder_->Stop();
8388}
8389
8390TEST_F(VideoStreamEncoderTest,
8391 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8392 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8393
8394 // Disable scaling settings in encoder info.
8395 fake_encoder_.SetQualityScaling(false);
8396 // Set QP trusted in encoder info.
8397 fake_encoder_.SetIsQpTrusted(false);
8398 // Enable quality scaling in encoder config.
8399 video_encoder_config.is_quality_scaling_allowed = true;
8400 ConfigureEncoder(std::move(video_encoder_config));
8401
8402 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008403 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008404
8405 test::FrameForwarder source;
8406 video_stream_encoder_->SetSource(
8407 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8408 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8409 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8410
8411 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8412 WaitForEncodedFrame(1);
8413 video_stream_encoder_->TriggerQualityLow();
8414 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8415
8416 video_stream_encoder_->Stop();
8417}
8418
Sergey Silkind19e3b92021-03-16 10:05:30 +00008419#endif
8420
Henrik Boström56db9ff2021-03-24 09:06:45 +01008421// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8422class VideoStreamEncoderWithRealEncoderTest
8423 : public VideoStreamEncoderTest,
8424 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8425 public:
8426 VideoStreamEncoderWithRealEncoderTest()
8427 : VideoStreamEncoderTest(),
8428 codec_type_(std::get<0>(GetParam())),
8429 allow_i420_conversion_(std::get<1>(GetParam())) {}
8430
8431 void SetUp() override {
8432 VideoStreamEncoderTest::SetUp();
8433 std::unique_ptr<VideoEncoder> encoder;
8434 switch (codec_type_) {
8435 case kVideoCodecVP8:
8436 encoder = VP8Encoder::Create();
8437 break;
8438 case kVideoCodecVP9:
8439 encoder = VP9Encoder::Create();
8440 break;
8441 case kVideoCodecAV1:
philipel95701502022-01-18 18:47:52 +01008442 encoder = CreateLibaomAv1EncoderIfSupported();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008443 break;
8444 case kVideoCodecH264:
8445 encoder =
8446 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8447 break;
8448 case kVideoCodecMultiplex:
8449 mock_encoder_factory_for_multiplex_ =
8450 std::make_unique<MockVideoEncoderFactory>();
8451 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8452 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8453 .WillRepeatedly([] { return VP8Encoder::Create(); });
8454 encoder = std::make_unique<MultiplexEncoderAdapter>(
8455 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8456 false);
8457 break;
8458 default:
Artem Titovd3251962021-11-15 16:57:07 +01008459 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008460 }
8461 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8462 }
8463
8464 void TearDown() override {
8465 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 16:22:11 +02008466 // Ensure `video_stream_encoder_` is destroyed before
8467 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 09:06:45 +01008468 video_stream_encoder_.reset();
8469 VideoStreamEncoderTest::TearDown();
8470 }
8471
8472 protected:
8473 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8474 std::unique_ptr<VideoEncoder> encoder) {
8475 // Configure VSE to use the encoder.
8476 encoder_ = std::move(encoder);
8477 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8478 encoder_.get(), &encoder_selector_);
8479 video_send_config_.encoder_settings.encoder_factory =
8480 encoder_proxy_factory_.get();
8481 VideoEncoderConfig video_encoder_config;
8482 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8483 video_encoder_config_ = video_encoder_config.Copy();
8484 ConfigureEncoder(video_encoder_config_.Copy());
8485
8486 // Set bitrate to ensure frame is not dropped.
8487 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008488 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008489 }
8490
8491 const VideoCodecType codec_type_;
8492 const bool allow_i420_conversion_;
8493 NiceMock<MockEncoderSelector> encoder_selector_;
8494 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8495 std::unique_ptr<VideoEncoder> encoder_;
8496 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8497};
8498
8499TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8500 auto native_i420_frame = test::CreateMappableNativeFrame(
8501 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8502 video_source_.IncomingCapturedFrame(native_i420_frame);
8503 WaitForEncodedFrame(codec_width_, codec_height_);
8504
8505 auto mappable_native_buffer =
8506 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8507 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8508 mappable_native_buffer->GetMappedFramedBuffers();
8509 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8510 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8511 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8512 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8513}
8514
8515TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8516 auto native_nv12_frame = test::CreateMappableNativeFrame(
8517 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8518 video_source_.IncomingCapturedFrame(native_nv12_frame);
8519 WaitForEncodedFrame(codec_width_, codec_height_);
8520
8521 auto mappable_native_buffer =
8522 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8523 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8524 mappable_native_buffer->GetMappedFramedBuffers();
8525 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8526 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8527 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8528 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8529
8530 if (!allow_i420_conversion_) {
8531 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8532 }
8533}
8534
Erik Språng7444b192021-06-02 14:02:13 +02008535TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8536 if (codec_type_ == kVideoCodecMultiplex) {
8537 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8538 return;
8539 }
8540
8541 const size_t kNumSpatialLayers = 3u;
8542 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8543 const int kFrameWidth = 1280;
8544 const int kFrameHeight = 720;
8545 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8546 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8547 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8548 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8549 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8550 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8551
8552 VideoEncoderConfig config;
8553 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8554 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008555 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008556 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8557 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8558 vp9_settings.numberOfTemporalLayers = 3;
8559 vp9_settings.automaticResizeOn = false;
8560 config.encoder_specific_settings =
8561 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8562 vp9_settings);
8563 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8564 /*fps=*/30.0,
8565 /*first_active_layer=*/0,
8566 /*num_spatial_layers=*/3,
8567 /*num_temporal_layers=*/3,
8568 /*is_screenshare=*/false);
8569 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8570 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008571 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008572 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8573 /*fps=*/30.0,
8574 /*first_active_layer=*/0,
8575 /*num_spatial_layers=*/3,
8576 /*num_temporal_layers=*/3,
8577 /*is_screenshare=*/false);
8578 config.simulcast_layers[0].scalability_mode = "L3T3_KEY";
8579 } else {
8580 // Simulcast for VP8/H264.
8581 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8582 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8583 config.simulcast_layers[i].scale_resolution_down_by =
8584 kDownscaleFactors[i];
8585 config.simulcast_layers[i].active = true;
8586 }
8587 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8588 // Turn off frame dropping to prevent flakiness.
8589 VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings();
8590 h264_settings.frameDroppingOn = false;
8591 config.encoder_specific_settings = rtc::make_ref_counted<
8592 VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings);
8593 }
8594 }
8595
8596 auto set_layer_active = [&](int layer_idx, bool active) {
8597 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8598 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8599 config.spatial_layers[layer_idx].active = active;
8600 } else {
8601 config.simulcast_layers[layer_idx].active = active;
8602 }
8603 };
8604
8605 config.video_stream_factory =
8606 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8607 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8608 /*screencast*/ false,
8609 /*screenshare enabled*/ false);
8610 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008611 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8612 0, 0, 0);
Erik Språng7444b192021-06-02 14:02:13 +02008613
8614 // Capture a frame with all layers active.
8615 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8616 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8617 int64_t timestamp_ms = kFrameIntervalMs;
8618 video_source_.IncomingCapturedFrame(
8619 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8620
8621 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8622 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8623
8624 // Capture a frame with one of the layers inactive.
8625 set_layer_active(2, false);
8626 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8627 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8628 timestamp_ms += kFrameIntervalMs;
8629 video_source_.IncomingCapturedFrame(
8630 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8631 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8632
8633 // New target bitrates signaled based on lower resolution.
8634 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8635 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8636 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8637 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8638
8639 // Re-enable the top layer.
8640 set_layer_active(2, true);
8641 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8642 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8643 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8644
8645 // Bitrate target adjusted back up to enable HD layer...
8646 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8647 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8648 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8649 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8650
8651 // ...then add a new frame.
8652 timestamp_ms += kFrameIntervalMs;
8653 video_source_.IncomingCapturedFrame(
8654 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8655 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8656 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8657
8658 video_stream_encoder_->Stop();
8659}
8660
Henrik Boström56db9ff2021-03-24 09:06:45 +01008661std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8662 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8663 VideoCodecType codec_type = std::get<0>(info.param);
8664 bool allow_i420_conversion = std::get<1>(info.param);
8665 std::string str;
8666 switch (codec_type) {
8667 case kVideoCodecGeneric:
8668 str = "Generic";
8669 break;
8670 case kVideoCodecVP8:
8671 str = "VP8";
8672 break;
8673 case kVideoCodecVP9:
8674 str = "VP9";
8675 break;
8676 case kVideoCodecAV1:
8677 str = "AV1";
8678 break;
8679 case kVideoCodecH264:
8680 str = "H264";
8681 break;
8682 case kVideoCodecMultiplex:
8683 str = "Multiplex";
8684 break;
8685 default:
Artem Titovd3251962021-11-15 16:57:07 +01008686 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008687 }
8688 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8689 return str;
8690}
8691
8692constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8693 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8694constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8695 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8696constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
8697 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/true);
8698constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8699 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8700#if defined(WEBRTC_USE_H264)
8701constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8702 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8703
8704// The windows compiler does not tolerate #if statements inside the
8705// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8706// and without H264).
8707INSTANTIATE_TEST_SUITE_P(
8708 All,
8709 VideoStreamEncoderWithRealEncoderTest,
8710 ::testing::Values(kVP8DisallowConversion,
8711 kVP9DisallowConversion,
8712 kAV1AllowConversion,
8713 kMultiplexDisallowConversion,
8714 kH264AllowConversion),
8715 TestParametersVideoCodecAndAllowI420ConversionToString);
8716#else
8717INSTANTIATE_TEST_SUITE_P(
8718 All,
8719 VideoStreamEncoderWithRealEncoderTest,
8720 ::testing::Values(kVP8DisallowConversion,
8721 kVP9DisallowConversion,
8722 kAV1AllowConversion,
8723 kMultiplexDisallowConversion),
8724 TestParametersVideoCodecAndAllowI420ConversionToString);
8725#endif
8726
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008727class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8728 protected:
8729 void RunTest(const std::vector<VideoStream>& configs,
8730 const int expected_num_init_encode) {
8731 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008732 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008733 InsertFrameAndWaitForEncoded();
8734 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8735 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008736 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8737 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008738
8739 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8740 ConfigureEncoder(configs[1]);
8741 InsertFrameAndWaitForEncoded();
8742 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8743 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008744 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008745 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 10:07:11 +02008746 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008747
8748 video_stream_encoder_->Stop();
8749 }
8750
8751 void ConfigureEncoder(const VideoStream& stream) {
8752 VideoEncoderConfig config;
8753 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8754 config.max_bitrate_bps = stream.max_bitrate_bps;
8755 config.simulcast_layers[0] = stream;
8756 config.video_stream_factory =
8757 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8758 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8759 /*conference_mode=*/false);
8760 video_stream_encoder_->ConfigureEncoder(std::move(config),
8761 kMaxPayloadLength);
8762 }
8763
8764 void OnBitrateUpdated(DataRate bitrate) {
8765 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8766 bitrate, bitrate, bitrate, 0, 0, 0);
8767 }
8768
8769 void InsertFrameAndWaitForEncoded() {
8770 timestamp_ms_ += kFrameIntervalMs;
8771 video_source_.IncomingCapturedFrame(
8772 CreateFrame(timestamp_ms_, kWidth, kHeight));
8773 sink_.WaitForEncodedFrame(timestamp_ms_);
8774 }
8775
8776 void ExpectEqual(const VideoCodec& actual,
8777 const VideoStream& expected) const {
8778 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8779 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8780 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
8781 static_cast<unsigned int>(expected.min_bitrate_bps));
8782 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
8783 static_cast<unsigned int>(expected.max_bitrate_bps));
8784 EXPECT_EQ(actual.simulcastStream[0].width,
8785 kWidth / expected.scale_resolution_down_by);
8786 EXPECT_EQ(actual.simulcastStream[0].height,
8787 kHeight / expected.scale_resolution_down_by);
8788 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
8789 expected.num_temporal_layers);
8790 EXPECT_EQ(actual.ScalabilityMode(), expected.scalability_mode);
8791 }
8792
8793 VideoStream DefaultConfig() const {
8794 VideoStream stream;
8795 stream.max_framerate = 25;
8796 stream.min_bitrate_bps = 35000;
8797 stream.max_bitrate_bps = 900000;
8798 stream.scale_resolution_down_by = 1.0;
8799 stream.num_temporal_layers = 1;
8800 stream.bitrate_priority = 1.0;
8801 stream.scalability_mode = "";
8802 return stream;
8803 }
8804
8805 const int kWidth = 640;
8806 const int kHeight = 360;
8807 int64_t timestamp_ms_ = 0;
8808};
8809
8810TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
8811 VideoStream config1 = DefaultConfig();
8812 VideoStream config2 = config1;
8813 config2.max_framerate++;
8814
8815 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8816}
8817
8818TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
8819 VideoStream config1 = DefaultConfig();
8820 VideoStream config2 = config1;
8821 config2.min_bitrate_bps += 10000;
8822
8823 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8824}
8825
8826TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
8827 VideoStream config1 = DefaultConfig();
8828 VideoStream config2 = config1;
8829 config2.max_bitrate_bps += 100000;
8830
8831 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8832}
8833
8834TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
8835 VideoStream config1 = DefaultConfig();
8836 VideoStream config2 = config1;
8837 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
8838
8839 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
8840}
8841
8842TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
8843 VideoStream config1 = DefaultConfig();
8844 VideoStream config2 = config1;
8845 config2.scale_resolution_down_by *= 2;
8846
8847 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8848}
8849
8850TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
8851 VideoStream config1 = DefaultConfig();
8852 VideoStream config2 = config1;
8853 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
8854
8855 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8856}
8857
8858TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
8859 VideoStream config1 = DefaultConfig();
8860 VideoStream config2 = config1;
8861 config2.scalability_mode = "L1T2";
8862
8863 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
8864}
8865
Tommi62b01db2022-01-25 23:41:22 +01008866// Simple test that just creates and then immediately destroys an encoder.
8867// The purpose of the test is to make sure that nothing bad happens if the
8868// initialization step on the encoder queue, doesn't run.
8869TEST(VideoStreamEncoderSimpleTest, CreateDestroy) {
8870 class SuperLazyTaskQueue : public webrtc::TaskQueueBase {
8871 public:
8872 SuperLazyTaskQueue() = default;
8873 ~SuperLazyTaskQueue() override = default;
8874
8875 private:
8876 void Delete() override { delete this; }
8877 void PostTask(std::unique_ptr<QueuedTask> task) override {
8878 // meh.
8879 }
8880 void PostDelayedTask(std::unique_ptr<QueuedTask> task,
8881 uint32_t milliseconds) override {
8882 ASSERT_TRUE(false);
8883 }
8884 };
8885
8886 // Lots of boiler plate.
8887 GlobalSimulatedTimeController time_controller(Timestamp::Millis(0));
8888 auto stats_proxy = std::make_unique<MockableSendStatisticsProxy>(
8889 time_controller.GetClock(), VideoSendStream::Config(nullptr),
8890 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo);
8891 SimpleVideoStreamEncoderFactory::MockFakeEncoder mock_fake_encoder(
8892 time_controller.GetClock());
8893 test::VideoEncoderProxyFactory encoder_factory(&mock_fake_encoder);
8894 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory =
8895 CreateBuiltinVideoBitrateAllocatorFactory();
8896 VideoStreamEncoderSettings encoder_settings{
8897 VideoEncoder::Capabilities(/*loss_notification=*/false)};
8898 encoder_settings.encoder_factory = &encoder_factory;
8899 encoder_settings.bitrate_allocator_factory = bitrate_allocator_factory.get();
8900
8901 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8902 EXPECT_CALL((*adapter.get()), Initialize).WillOnce(Return());
8903
8904 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
8905 encoder_queue(new SuperLazyTaskQueue());
8906
8907 // Construct a VideoStreamEncoder instance and let it go out of scope without
8908 // doing anything else (including calling Stop()). This should be fine since
8909 // the posted init task will simply be deleted.
8910 auto encoder = std::make_unique<VideoStreamEncoder>(
8911 time_controller.GetClock(), 1, stats_proxy.get(), encoder_settings,
8912 std::make_unique<CpuOveruseDetectorProxy>(stats_proxy.get()),
8913 std::move(adapter), std::move(encoder_queue),
8914 VideoStreamEncoder::BitrateAllocationCallbackType::
8915 kVideoBitrateAllocation);
8916}
8917
Markus Handellb4e96d42021-11-05 12:00:55 +01008918TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
8919 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8920 auto* adapter_ptr = adapter.get();
8921 SimpleVideoStreamEncoderFactory factory;
Markus Handell8d87c462021-12-16 11:37:16 +01008922 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8923 nullptr;
8924 EXPECT_CALL(*adapter_ptr, Initialize)
8925 .WillOnce(Invoke([&video_stream_encoder_callback](
8926 FrameCadenceAdapterInterface::Callback* callback) {
8927 video_stream_encoder_callback = callback;
8928 }));
8929 TaskQueueBase* encoder_queue = nullptr;
8930 auto video_stream_encoder =
8931 factory.Create(std::move(adapter), &encoder_queue);
Markus Handellb4e96d42021-11-05 12:00:55 +01008932
Markus Handelle59fee82021-12-23 09:29:23 +01008933 // First a call before we know the frame size and hence cannot compute the
8934 // number of simulcast layers.
8935 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
8936 &FrameCadenceAdapterInterface::
8937 ZeroHertzModeParams::num_simulcast_layers,
8938 Eq(0)))));
Markus Handellb4e96d42021-11-05 12:00:55 +01008939 VideoEncoderConfig config;
Markus Handell8d87c462021-12-16 11:37:16 +01008940 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
Markus Handellb4e96d42021-11-05 12:00:55 +01008941 config.content_type = VideoEncoderConfig::ContentType::kScreen;
8942 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
Markus Handelle59fee82021-12-23 09:29:23 +01008943 factory.DepleteTaskQueues();
8944
8945 // Then a call as we've computed the number of simulcast layers after a passed
8946 // frame.
8947 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
8948 &FrameCadenceAdapterInterface::
8949 ZeroHertzModeParams::num_simulcast_layers,
8950 Gt(0)))));
Markus Handell8d87c462021-12-16 11:37:16 +01008951 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handell9a478b52021-11-18 16:07:01 +01008952 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01008953 Mock::VerifyAndClearExpectations(adapter_ptr);
8954
Markus Handelle59fee82021-12-23 09:29:23 +01008955 // Expect a disabled zero-hertz mode after passing realtime video.
Markus Handell8d87c462021-12-16 11:37:16 +01008956 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Eq(absl::nullopt)));
Markus Handellb4e96d42021-11-05 12:00:55 +01008957 VideoEncoderConfig config2;
Markus Handell8d87c462021-12-16 11:37:16 +01008958 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config2);
Markus Handellb4e96d42021-11-05 12:00:55 +01008959 config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
8960 video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
Markus Handell8d87c462021-12-16 11:37:16 +01008961 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
Markus Handell9a478b52021-11-18 16:07:01 +01008962 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01008963}
8964
8965TEST(VideoStreamEncoderFrameCadenceTest,
8966 ForwardsFramesIntoFrameCadenceAdapter) {
8967 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8968 auto* adapter_ptr = adapter.get();
8969 test::FrameForwarder video_source;
8970 SimpleVideoStreamEncoderFactory factory;
8971 auto video_stream_encoder = factory.Create(std::move(adapter));
8972 video_stream_encoder->SetSource(
8973 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8974
8975 EXPECT_CALL(*adapter_ptr, OnFrame);
8976 auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
8977 video_source.IncomingCapturedFrame(
Markus Handell8d87c462021-12-16 11:37:16 +01008978 VideoFrame::Builder().set_video_frame_buffer(std::move(buffer)).build());
Markus Handellb4e96d42021-11-05 12:00:55 +01008979}
8980
Markus Handellee225432021-11-29 12:35:12 +01008981TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) {
8982 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
8983 auto* adapter_ptr = adapter.get();
8984 test::FrameForwarder video_source;
8985 SimpleVideoStreamEncoderFactory factory;
8986 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
8987 nullptr;
8988 EXPECT_CALL(*adapter_ptr, Initialize)
8989 .WillOnce(Invoke([&video_stream_encoder_callback](
8990 FrameCadenceAdapterInterface::Callback* callback) {
8991 video_stream_encoder_callback = callback;
8992 }));
8993 TaskQueueBase* encoder_queue = nullptr;
8994 auto video_stream_encoder =
8995 factory.Create(std::move(adapter), &encoder_queue);
8996
8997 // This is just to make the VSE operational. We'll feed a frame directly by
8998 // the callback interface.
8999 video_stream_encoder->SetSource(
9000 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9001
9002 VideoEncoderConfig video_encoder_config;
9003 test::FillEncoderConfiguration(kVideoCodecGeneric, 1, &video_encoder_config);
9004 video_stream_encoder->ConfigureEncoder(std::move(video_encoder_config),
9005 /*max_data_payload_length=*/1000);
9006
9007 EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps);
9008 EXPECT_CALL(*adapter_ptr, UpdateFrameRate);
Markus Handell8d87c462021-12-16 11:37:16 +01009009 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handellee225432021-11-29 12:35:12 +01009010 factory.DepleteTaskQueues();
9011}
9012
Markus Handell8d87c462021-12-16 11:37:16 +01009013TEST(VideoStreamEncoderFrameCadenceTest,
9014 DeactivatesActivatesLayersOnBitrateChanges) {
9015 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9016 auto* adapter_ptr = adapter.get();
9017 SimpleVideoStreamEncoderFactory factory;
9018 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9019 nullptr;
9020 EXPECT_CALL(*adapter_ptr, Initialize)
9021 .WillOnce(Invoke([&video_stream_encoder_callback](
9022 FrameCadenceAdapterInterface::Callback* callback) {
9023 video_stream_encoder_callback = callback;
9024 }));
9025 TaskQueueBase* encoder_queue = nullptr;
9026 auto video_stream_encoder =
9027 factory.Create(std::move(adapter), &encoder_queue);
9028
9029 // Configure 2 simulcast layers. FillEncoderConfiguration sets min bitrates to
9030 // {150000, 450000}.
9031 VideoEncoderConfig video_encoder_config;
9032 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9033 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9034 kMaxPayloadLength);
9035 // Ensure an encoder is created.
9036 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9037
9038 // Both layers enabled at 1 MBit/s.
9039 video_stream_encoder->OnBitrateUpdated(
9040 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9041 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9042 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9043 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9044 factory.DepleteTaskQueues();
9045 Mock::VerifyAndClearExpectations(adapter_ptr);
9046
9047 // Layer 1 disabled at 200 KBit/s.
9048 video_stream_encoder->OnBitrateUpdated(
9049 DataRate::KilobitsPerSec(200), DataRate::KilobitsPerSec(200),
9050 DataRate::KilobitsPerSec(200), 0, 0, 0);
9051 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9052 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9053 factory.DepleteTaskQueues();
9054 Mock::VerifyAndClearExpectations(adapter_ptr);
9055
9056 // All layers off at suspended video.
9057 video_stream_encoder->OnBitrateUpdated(DataRate::Zero(), DataRate::Zero(),
9058 DataRate::Zero(), 0, 0, 0);
9059 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/false));
9060 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9061 factory.DepleteTaskQueues();
9062 Mock::VerifyAndClearExpectations(adapter_ptr);
9063
9064 // Both layers enabled again back at 1 MBit/s.
9065 video_stream_encoder->OnBitrateUpdated(
9066 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9067 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9068 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9069 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9070 factory.DepleteTaskQueues();
9071}
9072
9073TEST(VideoStreamEncoderFrameCadenceTest, UpdatesQualityConvergence) {
9074 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9075 auto* adapter_ptr = adapter.get();
9076 SimpleVideoStreamEncoderFactory factory;
9077 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9078 nullptr;
9079 EXPECT_CALL(*adapter_ptr, Initialize)
9080 .WillOnce(Invoke([&video_stream_encoder_callback](
9081 FrameCadenceAdapterInterface::Callback* callback) {
9082 video_stream_encoder_callback = callback;
9083 }));
9084 TaskQueueBase* encoder_queue = nullptr;
9085 auto video_stream_encoder =
9086 factory.Create(std::move(adapter), &encoder_queue);
9087
9088 // Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder.
9089 VideoEncoderConfig video_encoder_config;
9090 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9091 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9092 kMaxPayloadLength);
9093 video_stream_encoder->OnBitrateUpdated(
9094 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9095 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9096
9097 // Pass a frame which has unconverged results.
9098 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9099 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9100 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9101 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
9102 EXPECT_FALSE(encoded_image.IsAtTargetQuality());
9103 CodecSpecificInfo codec_specific;
9104 codec_specific.codecType = kVideoCodecGeneric;
9105 return codec_specific;
9106 }));
9107 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, false));
9108 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9109 factory.DepleteTaskQueues();
9110 Mock::VerifyAndClearExpectations(adapter_ptr);
9111 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9112
9113 // Pass a frame which converges in layer 0 and not in layer 1.
9114 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9115 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9116 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9117 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
9118 encoded_image.SetAtTargetQuality(encoded_image.SpatialIndex() == 0);
9119 CodecSpecificInfo codec_specific;
9120 codec_specific.codecType = kVideoCodecGeneric;
9121 return codec_specific;
9122 }));
9123 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, true));
9124 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9125 factory.DepleteTaskQueues();
9126 Mock::VerifyAndClearExpectations(adapter_ptr);
9127 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9128}
9129
Markus Handell2e0f4f02021-12-21 19:14:58 +01009130TEST(VideoStreamEncoderFrameCadenceTest,
9131 RequestsRefreshFramesWhenCadenceAdapterInstructs) {
9132 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9133 auto* adapter_ptr = adapter.get();
9134 MockVideoSourceInterface mock_source;
9135 SimpleVideoStreamEncoderFactory factory;
9136 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9137 nullptr;
9138 EXPECT_CALL(*adapter_ptr, Initialize)
9139 .WillOnce(Invoke([&video_stream_encoder_callback](
9140 FrameCadenceAdapterInterface::Callback* callback) {
9141 video_stream_encoder_callback = callback;
9142 }));
9143 TaskQueueBase* encoder_queue = nullptr;
9144 auto video_stream_encoder =
9145 factory.Create(std::move(adapter), &encoder_queue);
9146 video_stream_encoder->SetSource(
9147 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9148 VideoEncoderConfig config;
9149 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9150 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9151 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9152 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9153 // Ensure the encoder is set up.
9154 factory.DepleteTaskQueues();
9155
Markus Handell818e7fb2021-12-30 13:01:33 +01009156 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest)
9157 .WillOnce(Invoke([video_stream_encoder_callback] {
9158 video_stream_encoder_callback->RequestRefreshFrame();
9159 }));
Markus Handell2e0f4f02021-12-21 19:14:58 +01009160 EXPECT_CALL(mock_source, RequestRefreshFrame);
9161 video_stream_encoder->SendKeyFrame();
9162 factory.DepleteTaskQueues();
9163 Mock::VerifyAndClearExpectations(adapter_ptr);
9164 Mock::VerifyAndClearExpectations(&mock_source);
9165
Markus Handell818e7fb2021-12-30 13:01:33 +01009166 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest);
Markus Handell2e0f4f02021-12-21 19:14:58 +01009167 EXPECT_CALL(mock_source, RequestRefreshFrame).Times(0);
9168 video_stream_encoder->SendKeyFrame();
9169 factory.DepleteTaskQueues();
9170}
9171
Markus Handell818e7fb2021-12-30 13:01:33 +01009172TEST(VideoStreamEncoderFrameCadenceTest,
9173 RequestsRefreshFrameForEarlyZeroHertzKeyFrameRequest) {
9174 SimpleVideoStreamEncoderFactory factory;
9175 auto encoder_queue =
9176 factory.GetTimeController()->GetTaskQueueFactory()->CreateTaskQueue(
9177 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
9178
9179 // Enables zero-hertz mode.
9180 test::ScopedFieldTrials field_trials("WebRTC-ZeroHertzScreenshare/Enabled/");
9181 auto adapter = FrameCadenceAdapterInterface::Create(
9182 factory.GetTimeController()->GetClock(), encoder_queue.get());
9183 FrameCadenceAdapterInterface* adapter_ptr = adapter.get();
9184
9185 MockVideoSourceInterface mock_source;
9186 auto video_stream_encoder = factory.CreateWithEncoderQueue(
9187 std::move(adapter), std::move(encoder_queue));
9188
9189 video_stream_encoder->SetSource(
9190 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9191 VideoEncoderConfig config;
9192 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9193 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9194 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9195
9196 // Eventually expect a refresh frame request when requesting a key frame
9197 // before initializing zero-hertz mode. This can happen in reality because the
9198 // threads invoking key frame requests and constraints setup aren't
9199 // synchronized.
9200 EXPECT_CALL(mock_source, RequestRefreshFrame);
9201 video_stream_encoder->SendKeyFrame();
9202 adapter_ptr->OnConstraintsChanged(VideoTrackSourceConstraints{0, 30});
9203 factory.DepleteTaskQueues();
9204}
9205
perkj26091b12016-09-01 01:17:40 -07009206} // namespace webrtc