blob: 4082fd2dbf4dcc5abcd137d1e87b00979b75e8a9 [file] [log] [blame]
perkj26091b12016-09-01 01:17:40 -07001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Erik Språng4529fbc2018-10-12 10:30:31 +020011#include "video/video_stream_encoder.h"
12
sprangfe627f32017-03-29 08:24:59 -070013#include <algorithm>
perkj803d97f2016-11-01 11:45:46 -070014#include <limits>
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020015#include <memory>
Henrik Boström56db9ff2021-03-24 09:06:45 +010016#include <tuple>
Per512ecb32016-09-23 15:52:06 +020017#include <utility>
18
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020019#include "absl/memory/memory.h"
Markus Handell9a478b52021-11-18 16:07:01 +010020#include "api/rtp_parameters.h"
Danil Chapovalovd3ba2362019-04-10 17:01:23 +020021#include "api/task_queue/default_task_queue_factory.h"
Markus Handell818e7fb2021-12-30 13:01:33 +010022#include "api/task_queue/task_queue_base.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010023#include "api/task_queue/task_queue_factory.h"
Elad Alon45befc52019-07-02 11:20:09 +020024#include "api/test/mock_fec_controller_override.h"
philipel9b058032020-02-10 11:30:00 +010025#include "api/test/mock_video_encoder.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010026#include "api/test/mock_video_encoder_factory.h"
Markus Handell8d87c462021-12-16 11:37:16 +010027#include "api/units/data_rate.h"
Jiawei Ouc2ebe212018-11-08 10:02:56 -080028#include "api/video/builtin_video_bitrate_allocator_factory.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020029#include "api/video/i420_buffer.h"
Evan Shrubsole895556e2020-10-05 09:15:13 +020030#include "api/video/nv12_buffer.h"
Evan Shrubsolece0a11d2020-04-16 11:36:55 +020031#include "api/video/video_adaptation_reason.h"
Erik Språngf93eda12019-01-16 17:10:57 +010032#include "api/video/video_bitrate_allocation.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010033#include "api/video_codecs/sdp_video_format.h"
Erik Språnge4589cb2022-04-06 16:44:30 +020034#include "api/video_codecs/video_codec.h"
Elad Alon370f93a2019-06-11 14:57:57 +020035#include "api/video_codecs/video_encoder.h"
Erik Språng4529fbc2018-10-12 10:30:31 +020036#include "api/video_codecs/vp8_temporal_layers.h"
Elad Aloncde8ab22019-03-20 11:56:20 +010037#include "api/video_codecs/vp8_temporal_layers_factory.h"
Henrik Boström0f0aa9c2020-06-02 13:02:36 +020038#include "call/adaptation/test/fake_adaptation_constraint.h"
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +010039#include "call/adaptation/test/fake_resource.h"
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +020040#include "common_video/h264/h264_common.h"
Noah Richards51db4212019-06-12 06:59:12 -070041#include "common_video/include/video_frame_buffer.h"
Steve Anton10542f22019-01-11 09:11:00 -080042#include "media/base/video_adapter.h"
Åsa Perssonc5a74ff2020-09-20 17:50:00 +020043#include "media/engine/webrtc_video_engine.h"
philipel95701502022-01-18 18:47:52 +010044#include "modules/video_coding/codecs/av1/libaom_av1_encoder_supported.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010045#include "modules/video_coding/codecs/h264/include/h264.h"
46#include "modules/video_coding/codecs/multiplex/include/multiplex_encoder_adapter.h"
47#include "modules/video_coding/codecs/vp8/include/vp8.h"
48#include "modules/video_coding/codecs/vp9/include/vp9.h"
Sergey Silkin86684962018-03-28 19:32:37 +020049#include "modules/video_coding/codecs/vp9/include/vp9_globals.h"
Erik Språng7444b192021-06-02 14:02:13 +020050#include "modules/video_coding/codecs/vp9/svc_config.h"
Henrik Boström91aa7322020-04-28 12:24:33 +020051#include "modules/video_coding/utility/quality_scaler.h"
Åsa Perssonc29cb2c2019-03-25 12:06:59 +010052#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020053#include "rtc_base/event.h"
Åsa Persson258e9892021-02-25 10:39:51 +010054#include "rtc_base/experiments/encoder_info_settings.h"
Henrik Boström2671dac2020-05-19 16:29:09 +020055#include "rtc_base/gunit.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020056#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080057#include "rtc_base/ref_counted_object.h"
Markus Handella3765182020-07-08 13:13:32 +020058#include "rtc_base/synchronization/mutex.h"
Mirko Bonadei17f48782018-09-28 08:51:10 +020059#include "system_wrappers/include/metrics.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020060#include "test/encoder_settings.h"
61#include "test/fake_encoder.h"
Artem Titov33f9d2b2019-12-05 15:59:00 +010062#include "test/frame_forwarder.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020063#include "test/gmock.h"
64#include "test/gtest.h"
Henrik Boström56db9ff2021-03-24 09:06:45 +010065#include "test/mappable_native_buffer.h"
Jonas Orelandc7f691a2022-03-09 15:12:07 +010066#include "test/scoped_key_value_config.h"
Tomas Gunnarsson612445e2020-09-21 14:31:23 +020067#include "test/time_controller/simulated_time_controller.h"
Byoungchan Lee13fe3672022-04-06 10:44:42 +090068#include "test/video_encoder_nullable_proxy_factory.h"
Niels Möllercbcbc222018-09-28 09:07:24 +020069#include "test/video_encoder_proxy_factory.h"
Markus Handellb4e96d42021-11-05 12:00:55 +010070#include "video/frame_cadence_adapter.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020071#include "video/send_statistics_proxy.h"
perkj26091b12016-09-01 01:17:40 -070072
73namespace webrtc {
74
sprang57c2fff2017-01-16 06:24:02 -080075using ::testing::_;
philipeld9cc8c02019-09-16 14:53:40 +020076using ::testing::AllOf;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020077using ::testing::Eq;
philipeld9cc8c02019-09-16 14:53:40 +020078using ::testing::Field;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020079using ::testing::Ge;
80using ::testing::Gt;
Markus Handellee225432021-11-29 12:35:12 +010081using ::testing::Invoke;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +020082using ::testing::Le;
83using ::testing::Lt;
philipel9b058032020-02-10 11:30:00 +010084using ::testing::Matcher;
Markus Handellb4e96d42021-11-05 12:00:55 +010085using ::testing::Mock;
philipel9b058032020-02-10 11:30:00 +010086using ::testing::NiceMock;
Markus Handellb4e96d42021-11-05 12:00:55 +010087using ::testing::Optional;
philipel9b058032020-02-10 11:30:00 +010088using ::testing::Return;
Per Kjellander4190ce92020-12-15 17:24:55 +010089using ::testing::SizeIs;
philipeld9cc8c02019-09-16 14:53:40 +020090using ::testing::StrictMock;
kthelgason876222f2016-11-29 01:44:11 -080091
perkj803d97f2016-11-01 11:45:46 -070092namespace {
Åsa Persson8c1bf952018-09-13 10:42:19 +020093const int kMinPixelsPerFrame = 320 * 180;
Åsa Perssone644a032019-11-08 15:56:00 +010094const int kQpLow = 1;
95const int kQpHigh = 2;
Åsa Persson8c1bf952018-09-13 10:42:19 +020096const int kMinFramerateFps = 2;
97const int kMinBalancedFramerateFps = 7;
98const int64_t kFrameTimeoutMs = 100;
asapersson5f7226f2016-11-25 04:37:00 -080099const size_t kMaxPayloadLength = 1440;
Asa Persson606d3cb2021-10-04 10:07:11 +0200100const DataRate kTargetBitrate = DataRate::KilobitsPerSec(1000);
101const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(100);
102const DataRate kStartBitrate = DataRate::KilobitsPerSec(600);
103const DataRate kSimulcastTargetBitrate = DataRate::KilobitsPerSec(3150);
kthelgason2bc68642017-02-07 07:02:22 -0800104const int kMaxInitialFramedrop = 4;
sprangfda496a2017-06-15 04:21:07 -0700105const int kDefaultFramerate = 30;
Åsa Persson8c1bf952018-09-13 10:42:19 +0200106const int64_t kFrameIntervalMs = rtc::kNumMillisecsPerSec / kDefaultFramerate;
Niels Möllerfe407b72019-09-10 10:48:48 +0200107const int64_t kProcessIntervalMs = 1000;
Sergey Silkin41c650b2019-10-14 13:12:19 +0200108const VideoEncoder::ResolutionBitrateLimits
109 kEncoderBitrateLimits540p(960 * 540, 100 * 1000, 100 * 1000, 2000 * 1000);
110const VideoEncoder::ResolutionBitrateLimits
111 kEncoderBitrateLimits720p(1280 * 720, 200 * 1000, 200 * 1000, 4000 * 1000);
asapersson5f7226f2016-11-25 04:37:00 -0800112
Asa Persson606d3cb2021-10-04 10:07:11 +0200113uint8_t kOptimalSps[] = {0, 0, 0, 1, H264::NaluType::kSps,
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +0200114 0x00, 0x00, 0x03, 0x03, 0xF4,
115 0x05, 0x03, 0xC7, 0xE0, 0x1B,
116 0x41, 0x10, 0x8D, 0x00};
117
Sergey Silkin0e42cf72021-03-15 10:12:57 +0100118const uint8_t kCodedFrameVp8Qp25[] = {
119 0x10, 0x02, 0x00, 0x9d, 0x01, 0x2a, 0x10, 0x00, 0x10, 0x00,
120 0x02, 0x47, 0x08, 0x85, 0x85, 0x88, 0x85, 0x84, 0x88, 0x0c,
121 0x82, 0x00, 0x0c, 0x0d, 0x60, 0x00, 0xfe, 0xfc, 0x5c, 0xd0};
122
Markus Handell818e7fb2021-12-30 13:01:33 +0100123VideoFrame CreateSimpleNV12Frame() {
124 return VideoFrame::Builder()
125 .set_video_frame_buffer(rtc::make_ref_counted<NV12Buffer>(
126 /*width=*/16, /*height=*/16))
127 .build();
128}
129
Markus Handell8d87c462021-12-16 11:37:16 +0100130void PassAFrame(
131 TaskQueueBase* encoder_queue,
132 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback,
133 int64_t ntp_time_ms) {
134 encoder_queue->PostTask(
135 ToQueuedTask([video_stream_encoder_callback, ntp_time_ms] {
Markus Handell818e7fb2021-12-30 13:01:33 +0100136 video_stream_encoder_callback->OnFrame(Timestamp::Millis(ntp_time_ms),
137 1, CreateSimpleNV12Frame());
Markus Handell8d87c462021-12-16 11:37:16 +0100138 }));
139}
140
perkj803d97f2016-11-01 11:45:46 -0700141class TestBuffer : public webrtc::I420Buffer {
142 public:
143 TestBuffer(rtc::Event* event, int width, int height)
144 : I420Buffer(width, height), event_(event) {}
145
146 private:
147 friend class rtc::RefCountedObject<TestBuffer>;
148 ~TestBuffer() override {
149 if (event_)
150 event_->Set();
151 }
152 rtc::Event* const event_;
153};
154
Henrik Boström56db9ff2021-03-24 09:06:45 +0100155// A fake native buffer that can't be converted to I420. Upon scaling, it
156// produces another FakeNativeBuffer.
Noah Richards51db4212019-06-12 06:59:12 -0700157class FakeNativeBuffer : public webrtc::VideoFrameBuffer {
158 public:
159 FakeNativeBuffer(rtc::Event* event, int width, int height)
160 : event_(event), width_(width), height_(height) {}
161 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
162 int width() const override { return width_; }
163 int height() const override { return height_; }
164 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
165 return nullptr;
166 }
Henrik Boström56db9ff2021-03-24 09:06:45 +0100167 rtc::scoped_refptr<VideoFrameBuffer> CropAndScale(
168 int offset_x,
169 int offset_y,
170 int crop_width,
171 int crop_height,
172 int scaled_width,
173 int scaled_height) override {
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200174 return rtc::make_ref_counted<FakeNativeBuffer>(nullptr, scaled_width,
175 scaled_height);
Henrik Boström56db9ff2021-03-24 09:06:45 +0100176 }
Noah Richards51db4212019-06-12 06:59:12 -0700177
178 private:
179 friend class rtc::RefCountedObject<FakeNativeBuffer>;
180 ~FakeNativeBuffer() override {
181 if (event_)
182 event_->Set();
183 }
184 rtc::Event* const event_;
185 const int width_;
186 const int height_;
187};
188
Evan Shrubsole895556e2020-10-05 09:15:13 +0200189// A fake native buffer that is backed by an NV12 buffer.
190class FakeNV12NativeBuffer : public webrtc::VideoFrameBuffer {
191 public:
192 FakeNV12NativeBuffer(rtc::Event* event, int width, int height)
193 : nv12_buffer_(NV12Buffer::Create(width, height)), event_(event) {}
194
195 webrtc::VideoFrameBuffer::Type type() const override { return Type::kNative; }
196 int width() const override { return nv12_buffer_->width(); }
197 int height() const override { return nv12_buffer_->height(); }
198 rtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
199 return nv12_buffer_->ToI420();
200 }
Evan Shrubsoleb556b082020-10-08 14:56:45 +0200201 rtc::scoped_refptr<VideoFrameBuffer> GetMappedFrameBuffer(
202 rtc::ArrayView<VideoFrameBuffer::Type> types) override {
203 if (absl::c_find(types, Type::kNV12) != types.end()) {
204 return nv12_buffer_;
205 }
206 return nullptr;
207 }
Niels Möllerba2de582022-04-20 16:46:26 +0200208 const NV12BufferInterface* GetNV12() const { return nv12_buffer_.get(); }
Evan Shrubsole895556e2020-10-05 09:15:13 +0200209
210 private:
211 friend class rtc::RefCountedObject<FakeNV12NativeBuffer>;
212 ~FakeNV12NativeBuffer() override {
213 if (event_)
214 event_->Set();
215 }
216 rtc::scoped_refptr<NV12Buffer> nv12_buffer_;
217 rtc::Event* const event_;
218};
219
Niels Möller7dc26b72017-12-06 10:27:48 +0100220class CpuOveruseDetectorProxy : public OveruseFrameDetector {
221 public:
Niels Möllerd1f7eb62018-03-28 16:40:58 +0200222 explicit CpuOveruseDetectorProxy(CpuOveruseMetricsObserver* metrics_observer)
223 : OveruseFrameDetector(metrics_observer),
Henrik Boström381d1092020-05-12 18:49:07 +0200224 last_target_framerate_fps_(-1),
225 framerate_updated_event_(true /* manual_reset */,
226 false /* initially_signaled */) {}
Niels Möller7dc26b72017-12-06 10:27:48 +0100227 virtual ~CpuOveruseDetectorProxy() {}
228
229 void OnTargetFramerateUpdated(int framerate_fps) override {
Markus Handella3765182020-07-08 13:13:32 +0200230 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100231 last_target_framerate_fps_ = framerate_fps;
232 OveruseFrameDetector::OnTargetFramerateUpdated(framerate_fps);
Henrik Boström381d1092020-05-12 18:49:07 +0200233 framerate_updated_event_.Set();
Niels Möller7dc26b72017-12-06 10:27:48 +0100234 }
235
236 int GetLastTargetFramerate() {
Markus Handella3765182020-07-08 13:13:32 +0200237 MutexLock lock(&lock_);
Niels Möller7dc26b72017-12-06 10:27:48 +0100238 return last_target_framerate_fps_;
239 }
240
Niels Möller4db138e2018-04-19 09:04:13 +0200241 CpuOveruseOptions GetOptions() { return options_; }
242
Henrik Boström381d1092020-05-12 18:49:07 +0200243 rtc::Event* framerate_updated_event() { return &framerate_updated_event_; }
244
Niels Möller7dc26b72017-12-06 10:27:48 +0100245 private:
Markus Handella3765182020-07-08 13:13:32 +0200246 Mutex lock_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100247 int last_target_framerate_fps_ RTC_GUARDED_BY(lock_);
Henrik Boström381d1092020-05-12 18:49:07 +0200248 rtc::Event framerate_updated_event_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100249};
250
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200251class FakeVideoSourceRestrictionsListener
252 : public VideoSourceRestrictionsListener {
Henrik Boström381d1092020-05-12 18:49:07 +0200253 public:
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200254 FakeVideoSourceRestrictionsListener()
Henrik Boström381d1092020-05-12 18:49:07 +0200255 : was_restrictions_updated_(false), restrictions_updated_event_() {}
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200256 ~FakeVideoSourceRestrictionsListener() override {
Henrik Boström381d1092020-05-12 18:49:07 +0200257 RTC_DCHECK(was_restrictions_updated_);
258 }
259
260 rtc::Event* restrictions_updated_event() {
261 return &restrictions_updated_event_;
262 }
263
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200264 // VideoSourceRestrictionsListener implementation.
Henrik Boström381d1092020-05-12 18:49:07 +0200265 void OnVideoSourceRestrictionsUpdated(
266 VideoSourceRestrictions restrictions,
267 const VideoAdaptationCounters& adaptation_counters,
Evan Shrubsoleec0af262020-07-01 11:47:46 +0200268 rtc::scoped_refptr<Resource> reason,
269 const VideoSourceRestrictions& unfiltered_restrictions) override {
Henrik Boström381d1092020-05-12 18:49:07 +0200270 was_restrictions_updated_ = true;
271 restrictions_updated_event_.Set();
272 }
273
274 private:
275 bool was_restrictions_updated_;
276 rtc::Event restrictions_updated_event_;
277};
278
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200279auto WantsFps(Matcher<int> fps_matcher) {
280 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
281 fps_matcher);
282}
283
284auto WantsMaxPixels(Matcher<int> max_pixel_matcher) {
285 return Field("max_pixel_count", &rtc::VideoSinkWants::max_pixel_count,
286 AllOf(max_pixel_matcher, Gt(0)));
287}
288
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200289auto ResolutionMax() {
290 return AllOf(
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200291 WantsMaxPixels(Eq(std::numeric_limits<int>::max())),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200292 Field("target_pixel_count", &rtc::VideoSinkWants::target_pixel_count,
293 Eq(absl::nullopt)));
294}
295
296auto FpsMax() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200297 return WantsFps(Eq(kDefaultFramerate));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200298}
299
300auto FpsUnlimited() {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200301 return WantsFps(Eq(std::numeric_limits<int>::max()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200302}
303
304auto FpsMatchesResolutionMax(Matcher<int> fps_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200305 return AllOf(WantsFps(fps_matcher), ResolutionMax());
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200306}
307
308auto FpsMaxResolutionMatches(Matcher<int> pixel_matcher) {
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200309 return AllOf(FpsMax(), WantsMaxPixels(pixel_matcher));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200310}
311
312auto FpsMaxResolutionMax() {
313 return AllOf(FpsMax(), ResolutionMax());
314}
315
316auto UnlimitedSinkWants() {
317 return AllOf(FpsUnlimited(), ResolutionMax());
318}
319
320auto FpsInRangeForPixelsInBalanced(int last_frame_pixels) {
321 Matcher<int> fps_range_matcher;
322
323 if (last_frame_pixels <= 320 * 240) {
324 fps_range_matcher = AllOf(Ge(7), Le(10));
Åsa Perssonc5a74ff2020-09-20 17:50:00 +0200325 } else if (last_frame_pixels <= 480 * 360) {
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +0200326 fps_range_matcher = AllOf(Ge(10), Le(15));
327 } else if (last_frame_pixels <= 640 * 480) {
328 fps_range_matcher = Ge(15);
329 } else {
330 fps_range_matcher = Eq(kDefaultFramerate);
331 }
332 return Field("max_framerate_fps", &rtc::VideoSinkWants::max_framerate_fps,
333 fps_range_matcher);
334}
335
Evan Shrubsole5fd40602020-05-25 16:19:54 +0200336auto FpsEqResolutionEqTo(const rtc::VideoSinkWants& other_wants) {
337 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
338 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
339}
340
341auto FpsMaxResolutionLt(const rtc::VideoSinkWants& other_wants) {
342 return AllOf(FpsMax(), WantsMaxPixels(Lt(other_wants.max_pixel_count)));
343}
344
345auto FpsMaxResolutionGt(const rtc::VideoSinkWants& other_wants) {
346 return AllOf(FpsMax(), WantsMaxPixels(Gt(other_wants.max_pixel_count)));
347}
348
349auto FpsLtResolutionEq(const rtc::VideoSinkWants& other_wants) {
350 return AllOf(WantsFps(Lt(other_wants.max_framerate_fps)),
351 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
352}
353
354auto FpsGtResolutionEq(const rtc::VideoSinkWants& other_wants) {
355 return AllOf(WantsFps(Gt(other_wants.max_framerate_fps)),
356 WantsMaxPixels(Eq(other_wants.max_pixel_count)));
357}
358
359auto FpsEqResolutionLt(const rtc::VideoSinkWants& other_wants) {
360 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
361 WantsMaxPixels(Lt(other_wants.max_pixel_count)));
362}
363
364auto FpsEqResolutionGt(const rtc::VideoSinkWants& other_wants) {
365 return AllOf(WantsFps(Eq(other_wants.max_framerate_fps)),
366 WantsMaxPixels(Gt(other_wants.max_pixel_count)));
367}
368
mflodmancc3d4422017-08-03 08:27:51 -0700369class VideoStreamEncoderUnderTest : public VideoStreamEncoder {
perkj803d97f2016-11-01 11:45:46 -0700370 public:
Markus Handell9a478b52021-11-18 16:07:01 +0100371 VideoStreamEncoderUnderTest(
372 TimeController* time_controller,
373 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter,
374 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
375 encoder_queue,
376 SendStatisticsProxy* stats_proxy,
377 const VideoStreamEncoderSettings& settings,
378 VideoStreamEncoder::BitrateAllocationCallbackType
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100379 allocation_callback_type,
Erik Språnge4589cb2022-04-06 16:44:30 +0200380 const FieldTrialsView& field_trials,
381 int num_cores)
Markus Handell9a478b52021-11-18 16:07:01 +0100382 : VideoStreamEncoder(time_controller->GetClock(),
Erik Språnge4589cb2022-04-06 16:44:30 +0200383 num_cores,
Markus Handell9a478b52021-11-18 16:07:01 +0100384 stats_proxy,
385 settings,
386 std::unique_ptr<OveruseFrameDetector>(
387 overuse_detector_proxy_ =
388 new CpuOveruseDetectorProxy(stats_proxy)),
389 std::move(cadence_adapter),
390 std::move(encoder_queue),
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100391 allocation_callback_type,
392 field_trials),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200393 time_controller_(time_controller),
Henrik Boström5cc28b02020-06-01 17:59:05 +0200394 fake_cpu_resource_(FakeResource::Create("FakeResource[CPU]")),
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200395 fake_quality_resource_(FakeResource::Create("FakeResource[QP]")),
Evan Shrubsoledc4d4222020-07-09 11:47:10 +0200396 fake_adaptation_constraint_("FakeAdaptationConstraint") {
Henrik Boströmc55516d2020-05-11 16:29:22 +0200397 InjectAdaptationResource(fake_quality_resource_,
Evan Shrubsolece0a11d2020-04-16 11:36:55 +0200398 VideoAdaptationReason::kQuality);
Henrik Boströmc55516d2020-05-11 16:29:22 +0200399 InjectAdaptationResource(fake_cpu_resource_, VideoAdaptationReason::kCpu);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200400 InjectAdaptationConstraint(&fake_adaptation_constraint_);
Evan Shrubsoleaa6fbc12020-02-25 16:26:01 +0100401 }
perkj803d97f2016-11-01 11:45:46 -0700402
Henrik Boström381d1092020-05-12 18:49:07 +0200403 void SetSourceAndWaitForRestrictionsUpdated(
404 rtc::VideoSourceInterface<VideoFrame>* source,
405 const DegradationPreference& degradation_preference) {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200406 FakeVideoSourceRestrictionsListener listener;
407 AddRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200408 SetSource(source, degradation_preference);
409 listener.restrictions_updated_event()->Wait(5000);
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200410 RemoveRestrictionsListenerForTesting(&listener);
Henrik Boström381d1092020-05-12 18:49:07 +0200411 }
412
413 void SetSourceAndWaitForFramerateUpdated(
414 rtc::VideoSourceInterface<VideoFrame>* source,
415 const DegradationPreference& degradation_preference) {
416 overuse_detector_proxy_->framerate_updated_event()->Reset();
417 SetSource(source, degradation_preference);
418 overuse_detector_proxy_->framerate_updated_event()->Wait(5000);
419 }
420
421 void OnBitrateUpdatedAndWaitForManagedResources(
422 DataRate target_bitrate,
423 DataRate stable_target_bitrate,
424 DataRate link_allocation,
425 uint8_t fraction_lost,
426 int64_t round_trip_time_ms,
427 double cwnd_reduce_ratio) {
428 OnBitrateUpdated(target_bitrate, stable_target_bitrate, link_allocation,
429 fraction_lost, round_trip_time_ms, cwnd_reduce_ratio);
430 // Bitrate is updated on the encoder queue.
431 WaitUntilTaskQueueIsIdle();
Henrik Boström381d1092020-05-12 18:49:07 +0200432 }
433
kthelgason2fc52542017-03-03 00:24:41 -0800434 // This is used as a synchronisation mechanism, to make sure that the
435 // encoder queue is not blocked before we start sending it frames.
436 void WaitUntilTaskQueueIsIdle() {
Markus Handell28c71802021-11-08 10:11:55 +0100437 time_controller_->AdvanceTime(TimeDelta::Zero());
kthelgason2fc52542017-03-03 00:24:41 -0800438 }
439
Henrik Boström91aa7322020-04-28 12:24:33 +0200440 // Triggers resource usage measurements on the fake CPU resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200441 void TriggerCpuOveruse() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200442 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200443 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200444 fake_cpu_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200445 event.Set();
446 });
447 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100448 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200449 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200450
Henrik Boström91aa7322020-04-28 12:24:33 +0200451 void TriggerCpuUnderuse() {
452 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200453 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200454 fake_cpu_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200455 event.Set();
456 });
457 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100458 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200459 }
kthelgason876222f2016-11-29 01:44:11 -0800460
Henrik Boström91aa7322020-04-28 12:24:33 +0200461 // Triggers resource usage measurements on the fake quality resource.
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200462 void TriggerQualityLow() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200463 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200464 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200465 fake_quality_resource_->SetUsageState(ResourceUsageState::kOveruse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200466 event.Set();
467 });
468 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100469 time_controller_->AdvanceTime(TimeDelta::Zero());
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200470 }
Åsa Perssonb67c44c2019-09-24 15:25:32 +0200471 void TriggerQualityHigh() {
Henrik Boström91aa7322020-04-28 12:24:33 +0200472 rtc::Event event;
Evan Shrubsole85728412020-08-25 13:12:12 +0200473 encoder_queue()->PostTask([this, &event] {
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200474 fake_quality_resource_->SetUsageState(ResourceUsageState::kUnderuse);
Henrik Boström91aa7322020-04-28 12:24:33 +0200475 event.Set();
476 });
477 ASSERT_TRUE(event.Wait(5000));
Markus Handell28c71802021-11-08 10:11:55 +0100478 time_controller_->AdvanceTime(TimeDelta::Zero());
Henrik Boström91aa7322020-04-28 12:24:33 +0200479 }
480
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200481 TimeController* const time_controller_;
Niels Möller7dc26b72017-12-06 10:27:48 +0100482 CpuOveruseDetectorProxy* overuse_detector_proxy_;
Henrik Boströmc55516d2020-05-11 16:29:22 +0200483 rtc::scoped_refptr<FakeResource> fake_cpu_resource_;
484 rtc::scoped_refptr<FakeResource> fake_quality_resource_;
Henrik Boström0f0aa9c2020-06-02 13:02:36 +0200485 FakeAdaptationConstraint fake_adaptation_constraint_;
perkj803d97f2016-11-01 11:45:46 -0700486};
487
Noah Richards51db4212019-06-12 06:59:12 -0700488// Simulates simulcast behavior and makes highest stream resolutions divisible
489// by 4.
490class CroppingVideoStreamFactory
491 : public VideoEncoderConfig::VideoStreamFactoryInterface {
492 public:
Åsa Persson17b29b92020-10-17 12:57:58 +0200493 CroppingVideoStreamFactory() {}
Noah Richards51db4212019-06-12 06:59:12 -0700494
495 private:
496 std::vector<VideoStream> CreateEncoderStreams(
497 int width,
498 int height,
499 const VideoEncoderConfig& encoder_config) override {
500 std::vector<VideoStream> streams = test::CreateVideoStreams(
501 width - width % 4, height - height % 4, encoder_config);
Noah Richards51db4212019-06-12 06:59:12 -0700502 return streams;
503 }
Noah Richards51db4212019-06-12 06:59:12 -0700504};
505
sprangb1ca0732017-02-01 08:38:12 -0800506class AdaptingFrameForwarder : public test::FrameForwarder {
507 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200508 explicit AdaptingFrameForwarder(TimeController* time_controller)
509 : time_controller_(time_controller), adaptation_enabled_(false) {}
asaperssonfab67072017-04-04 05:51:49 -0700510 ~AdaptingFrameForwarder() override {}
sprangb1ca0732017-02-01 08:38:12 -0800511
512 void set_adaptation_enabled(bool enabled) {
Markus Handella3765182020-07-08 13:13:32 +0200513 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800514 adaptation_enabled_ = enabled;
515 }
516
asaperssonfab67072017-04-04 05:51:49 -0700517 bool adaption_enabled() const {
Markus Handella3765182020-07-08 13:13:32 +0200518 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -0800519 return adaptation_enabled_;
520 }
521
Henrik Boström1124ed12021-02-25 10:30:39 +0100522 // The "last wants" is a snapshot of the previous rtc::VideoSinkWants where
523 // the resolution or frame rate was different than it is currently. If
524 // something else is modified, such as encoder resolutions, but the resolution
525 // and frame rate stays the same, last wants is not updated.
asapersson09f05612017-05-15 23:40:18 -0700526 rtc::VideoSinkWants last_wants() const {
Markus Handella3765182020-07-08 13:13:32 +0200527 MutexLock lock(&mutex_);
asapersson09f05612017-05-15 23:40:18 -0700528 return last_wants_;
529 }
530
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200531 absl::optional<int> last_sent_width() const { return last_width_; }
532 absl::optional<int> last_sent_height() const { return last_height_; }
Jonathan Yubc771b72017-12-08 17:04:29 -0800533
sprangb1ca0732017-02-01 08:38:12 -0800534 void IncomingCapturedFrame(const VideoFrame& video_frame) override {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200535 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +0100536 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200537
sprangb1ca0732017-02-01 08:38:12 -0800538 int cropped_width = 0;
539 int cropped_height = 0;
540 int out_width = 0;
541 int out_height = 0;
sprangc5d62e22017-04-02 23:53:04 -0700542 if (adaption_enabled()) {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000543 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: AdaptFrameResolution()"
544 << "w=" << video_frame.width()
545 << "h=" << video_frame.height();
sprangc5d62e22017-04-02 23:53:04 -0700546 if (adapter_.AdaptFrameResolution(
547 video_frame.width(), video_frame.height(),
548 video_frame.timestamp_us() * 1000, &cropped_width,
549 &cropped_height, &out_width, &out_height)) {
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100550 VideoFrame adapted_frame =
551 VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200552 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100553 nullptr, out_width, out_height))
Åsa Persson90719572021-04-08 19:05:30 +0200554 .set_ntp_time_ms(video_frame.ntp_time_ms())
Artem Titov1ebfb6a2019-01-03 23:49:37 +0100555 .set_timestamp_ms(99)
556 .set_rotation(kVideoRotation_0)
557 .build();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +0100558 if (video_frame.has_update_rect()) {
559 adapted_frame.set_update_rect(
560 video_frame.update_rect().ScaleWithFrame(
561 video_frame.width(), video_frame.height(), 0, 0,
562 video_frame.width(), video_frame.height(), out_width,
563 out_height));
564 }
sprangc5d62e22017-04-02 23:53:04 -0700565 test::FrameForwarder::IncomingCapturedFrame(adapted_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800566 last_width_.emplace(adapted_frame.width());
567 last_height_.emplace(adapted_frame.height());
568 } else {
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200569 last_width_ = absl::nullopt;
570 last_height_ = absl::nullopt;
sprangc5d62e22017-04-02 23:53:04 -0700571 }
sprangb1ca0732017-02-01 08:38:12 -0800572 } else {
Harald Alvestrand97597c02021-11-04 12:01:23 +0000573 RTC_DLOG(LS_INFO) << "IncomingCapturedFrame: adaptation not enabled";
sprangb1ca0732017-02-01 08:38:12 -0800574 test::FrameForwarder::IncomingCapturedFrame(video_frame);
Jonathan Yubc771b72017-12-08 17:04:29 -0800575 last_width_.emplace(video_frame.width());
576 last_height_.emplace(video_frame.height());
sprangb1ca0732017-02-01 08:38:12 -0800577 }
578 }
579
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +0200580 void OnOutputFormatRequest(int width, int height) {
581 absl::optional<std::pair<int, int>> target_aspect_ratio =
582 std::make_pair(width, height);
583 absl::optional<int> max_pixel_count = width * height;
584 absl::optional<int> max_fps;
585 adapter_.OnOutputFormatRequest(target_aspect_ratio, max_pixel_count,
586 max_fps);
587 }
588
sprangb1ca0732017-02-01 08:38:12 -0800589 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
590 const rtc::VideoSinkWants& wants) override {
Markus Handella3765182020-07-08 13:13:32 +0200591 MutexLock lock(&mutex_);
Henrik Boström1124ed12021-02-25 10:30:39 +0100592 rtc::VideoSinkWants prev_wants = sink_wants_locked();
593 bool did_adapt =
594 prev_wants.max_pixel_count != wants.max_pixel_count ||
595 prev_wants.target_pixel_count != wants.target_pixel_count ||
596 prev_wants.max_framerate_fps != wants.max_framerate_fps;
597 if (did_adapt) {
598 last_wants_ = prev_wants;
599 }
Rasmus Brandt287e4642019-11-15 16:56:01 +0100600 adapter_.OnSinkWants(wants);
Markus Handell16038ab2020-05-28 08:37:30 +0200601 test::FrameForwarder::AddOrUpdateSinkLocked(sink, wants);
sprangb1ca0732017-02-01 08:38:12 -0800602 }
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200603
604 TimeController* const time_controller_;
sprangb1ca0732017-02-01 08:38:12 -0800605 cricket::VideoAdapter adapter_;
Markus Handella3765182020-07-08 13:13:32 +0200606 bool adaptation_enabled_ RTC_GUARDED_BY(mutex_);
607 rtc::VideoSinkWants last_wants_ RTC_GUARDED_BY(mutex_);
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200608 absl::optional<int> last_width_;
609 absl::optional<int> last_height_;
sprangb1ca0732017-02-01 08:38:12 -0800610};
sprangc5d62e22017-04-02 23:53:04 -0700611
Niels Möller213618e2018-07-24 09:29:58 +0200612// TODO(nisse): Mock only VideoStreamEncoderObserver.
sprangc5d62e22017-04-02 23:53:04 -0700613class MockableSendStatisticsProxy : public SendStatisticsProxy {
614 public:
615 MockableSendStatisticsProxy(Clock* clock,
616 const VideoSendStream::Config& config,
Jonas Oreland8ca06132022-03-14 12:52:48 +0100617 VideoEncoderConfig::ContentType content_type,
Jonas Orelande62c2f22022-03-29 11:04:48 +0200618 const FieldTrialsView& field_trials)
Jonas Oreland8ca06132022-03-14 12:52:48 +0100619 : SendStatisticsProxy(clock, config, content_type, field_trials) {}
sprangc5d62e22017-04-02 23:53:04 -0700620
621 VideoSendStream::Stats GetStats() override {
Markus Handella3765182020-07-08 13:13:32 +0200622 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700623 if (mock_stats_)
624 return *mock_stats_;
625 return SendStatisticsProxy::GetStats();
626 }
627
Niels Möller213618e2018-07-24 09:29:58 +0200628 int GetInputFrameRate() const override {
Markus Handella3765182020-07-08 13:13:32 +0200629 MutexLock lock(&lock_);
Niels Möller213618e2018-07-24 09:29:58 +0200630 if (mock_stats_)
631 return mock_stats_->input_frame_rate;
632 return SendStatisticsProxy::GetInputFrameRate();
633 }
sprangc5d62e22017-04-02 23:53:04 -0700634 void SetMockStats(const VideoSendStream::Stats& stats) {
Markus Handella3765182020-07-08 13:13:32 +0200635 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700636 mock_stats_.emplace(stats);
637 }
638
639 void ResetMockStats() {
Markus Handella3765182020-07-08 13:13:32 +0200640 MutexLock lock(&lock_);
sprangc5d62e22017-04-02 23:53:04 -0700641 mock_stats_.reset();
642 }
643
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200644 void SetDroppedFrameCallback(std::function<void(DropReason)> callback) {
645 on_frame_dropped_ = std::move(callback);
646 }
647
sprangc5d62e22017-04-02 23:53:04 -0700648 private:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200649 void OnFrameDropped(DropReason reason) override {
650 SendStatisticsProxy::OnFrameDropped(reason);
651 if (on_frame_dropped_)
652 on_frame_dropped_(reason);
653 }
654
Markus Handella3765182020-07-08 13:13:32 +0200655 mutable Mutex lock_;
Danil Chapovalovb9b146c2018-06-15 12:28:07 +0200656 absl::optional<VideoSendStream::Stats> mock_stats_ RTC_GUARDED_BY(lock_);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200657 std::function<void(DropReason)> on_frame_dropped_;
sprangc5d62e22017-04-02 23:53:04 -0700658};
659
Markus Handellb4e96d42021-11-05 12:00:55 +0100660class SimpleVideoStreamEncoderFactory {
661 public:
662 class AdaptedVideoStreamEncoder : public VideoStreamEncoder {
663 public:
664 using VideoStreamEncoder::VideoStreamEncoder;
665 ~AdaptedVideoStreamEncoder() { Stop(); }
666 };
667
Markus Handell8d87c462021-12-16 11:37:16 +0100668 class MockFakeEncoder : public test::FakeEncoder {
669 public:
670 using FakeEncoder::FakeEncoder;
671 MOCK_METHOD(CodecSpecificInfo,
672 EncodeHook,
673 (EncodedImage & encoded_image,
674 rtc::scoped_refptr<EncodedImageBuffer> buffer),
675 (override));
676 };
677
Markus Handellee225432021-11-29 12:35:12 +0100678 SimpleVideoStreamEncoderFactory() {
Markus Handellb4e96d42021-11-05 12:00:55 +0100679 encoder_settings_.encoder_factory = &encoder_factory_;
Markus Handellee225432021-11-29 12:35:12 +0100680 encoder_settings_.bitrate_allocator_factory =
681 bitrate_allocator_factory_.get();
Markus Handellb4e96d42021-11-05 12:00:55 +0100682 }
683
Markus Handell818e7fb2021-12-30 13:01:33 +0100684 std::unique_ptr<AdaptedVideoStreamEncoder> CreateWithEncoderQueue(
Markus Handellee225432021-11-29 12:35:12 +0100685 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100686 std::unique_ptr<TaskQueueBase, TaskQueueDeleter> encoder_queue,
Jonas Orelande62c2f22022-03-29 11:04:48 +0200687 const FieldTrialsView* field_trials = nullptr) {
Markus Handellb4e96d42021-11-05 12:00:55 +0100688 auto result = std::make_unique<AdaptedVideoStreamEncoder>(
689 time_controller_.GetClock(),
690 /*number_of_cores=*/1,
691 /*stats_proxy=*/stats_proxy_.get(), encoder_settings_,
692 std::make_unique<CpuOveruseDetectorProxy>(/*stats_proxy=*/nullptr),
Markus Handellee225432021-11-29 12:35:12 +0100693 std::move(zero_hertz_adapter), std::move(encoder_queue),
Markus Handellb4e96d42021-11-05 12:00:55 +0100694 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100695 kVideoBitrateAllocation,
696 field_trials ? *field_trials : field_trials_);
Markus Handellb4e96d42021-11-05 12:00:55 +0100697 result->SetSink(&sink_, /*rotation_applied=*/false);
698 return result;
699 }
700
Markus Handell818e7fb2021-12-30 13:01:33 +0100701 std::unique_ptr<AdaptedVideoStreamEncoder> Create(
702 std::unique_ptr<FrameCadenceAdapterInterface> zero_hertz_adapter,
703 TaskQueueBase** encoder_queue_ptr = nullptr) {
704 auto encoder_queue =
705 time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
706 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
707 if (encoder_queue_ptr)
708 *encoder_queue_ptr = encoder_queue.get();
709 return CreateWithEncoderQueue(std::move(zero_hertz_adapter),
710 std::move(encoder_queue));
711 }
712
Markus Handell9a478b52021-11-18 16:07:01 +0100713 void DepleteTaskQueues() { time_controller_.AdvanceTime(TimeDelta::Zero()); }
Markus Handell8d87c462021-12-16 11:37:16 +0100714 MockFakeEncoder& GetMockFakeEncoder() { return mock_fake_encoder_; }
Markus Handell9a478b52021-11-18 16:07:01 +0100715
Markus Handell818e7fb2021-12-30 13:01:33 +0100716 GlobalSimulatedTimeController* GetTimeController() {
717 return &time_controller_;
718 }
719
Markus Handellb4e96d42021-11-05 12:00:55 +0100720 private:
721 class NullEncoderSink : public VideoStreamEncoderInterface::EncoderSink {
722 public:
723 ~NullEncoderSink() override = default;
724 void OnEncoderConfigurationChanged(
725 std::vector<VideoStream> streams,
726 bool is_svc,
727 VideoEncoderConfig::ContentType content_type,
728 int min_transmit_bitrate_bps) override {}
729 void OnBitrateAllocationUpdated(
730 const VideoBitrateAllocation& allocation) override {}
731 void OnVideoLayersAllocationUpdated(
732 VideoLayersAllocation allocation) override {}
733 Result OnEncodedImage(
734 const EncodedImage& encoded_image,
735 const CodecSpecificInfo* codec_specific_info) override {
736 return Result(EncodedImageCallback::Result::OK);
737 }
738 };
739
Jonas Orelandc7f691a2022-03-09 15:12:07 +0100740 test::ScopedKeyValueConfig field_trials_;
Markus Handellee225432021-11-29 12:35:12 +0100741 GlobalSimulatedTimeController time_controller_{Timestamp::Millis(0)};
742 std::unique_ptr<TaskQueueFactory> task_queue_factory_{
743 time_controller_.CreateTaskQueueFactory()};
744 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_ =
745 std::make_unique<MockableSendStatisticsProxy>(
746 time_controller_.GetClock(),
747 VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 12:52:48 +0100748 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
749 field_trials_);
Markus Handellee225432021-11-29 12:35:12 +0100750 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_ =
751 CreateBuiltinVideoBitrateAllocatorFactory();
752 VideoStreamEncoderSettings encoder_settings_{
753 VideoEncoder::Capabilities(/*loss_notification=*/false)};
Markus Handell8d87c462021-12-16 11:37:16 +0100754 MockFakeEncoder mock_fake_encoder_{time_controller_.GetClock()};
755 test::VideoEncoderProxyFactory encoder_factory_{&mock_fake_encoder_};
Markus Handellb4e96d42021-11-05 12:00:55 +0100756 NullEncoderSink sink_;
757};
758
759class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
760 public:
761 MOCK_METHOD(void, Initialize, (Callback * callback), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100762 MOCK_METHOD(void,
763 SetZeroHertzModeEnabled,
764 (absl::optional<ZeroHertzModeParams>),
765 (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100766 MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
Markus Handellee225432021-11-29 12:35:12 +0100767 MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override));
768 MOCK_METHOD(void, UpdateFrameRate, (), (override));
Markus Handell8d87c462021-12-16 11:37:16 +0100769 MOCK_METHOD(void,
770 UpdateLayerQualityConvergence,
771 (int spatial_index, bool converged),
772 (override));
773 MOCK_METHOD(void,
774 UpdateLayerStatus,
775 (int spatial_index, bool enabled),
776 (override));
Markus Handell818e7fb2021-12-30 13:01:33 +0100777 MOCK_METHOD(void, ProcessKeyFrameRequest, (), (override));
Markus Handellb4e96d42021-11-05 12:00:55 +0100778};
779
philipel9b058032020-02-10 11:30:00 +0100780class MockEncoderSelector
781 : public VideoEncoderFactory::EncoderSelectorInterface {
782 public:
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200783 MOCK_METHOD(void,
784 OnCurrentEncoder,
785 (const SdpVideoFormat& format),
786 (override));
787 MOCK_METHOD(absl::optional<SdpVideoFormat>,
788 OnAvailableBitrate,
789 (const DataRate& rate),
790 (override));
philipel6daa3042022-04-11 10:48:28 +0200791 MOCK_METHOD(absl::optional<SdpVideoFormat>,
792 OnResolutionChange,
793 (const RenderResolution& resolution),
794 (override));
Danil Chapovalov91fdc602020-05-14 19:17:51 +0200795 MOCK_METHOD(absl::optional<SdpVideoFormat>, OnEncoderBroken, (), (override));
philipel9b058032020-02-10 11:30:00 +0100796};
797
Markus Handell2e0f4f02021-12-21 19:14:58 +0100798class MockVideoSourceInterface : public rtc::VideoSourceInterface<VideoFrame> {
799 public:
800 MOCK_METHOD(void,
801 AddOrUpdateSink,
802 (rtc::VideoSinkInterface<VideoFrame>*,
803 const rtc::VideoSinkWants&),
804 (override));
805 MOCK_METHOD(void,
806 RemoveSink,
807 (rtc::VideoSinkInterface<VideoFrame>*),
808 (override));
809 MOCK_METHOD(void, RequestRefreshFrame, (), (override));
810};
811
perkj803d97f2016-11-01 11:45:46 -0700812} // namespace
813
mflodmancc3d4422017-08-03 08:27:51 -0700814class VideoStreamEncoderTest : public ::testing::Test {
perkj26091b12016-09-01 01:17:40 -0700815 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200816 static const int kDefaultTimeoutMs = 1000;
perkj26091b12016-09-01 01:17:40 -0700817
mflodmancc3d4422017-08-03 08:27:51 -0700818 VideoStreamEncoderTest()
perkj26091b12016-09-01 01:17:40 -0700819 : video_send_config_(VideoSendStream::Config(nullptr)),
perkjfa10b552016-10-02 23:45:26 -0700820 codec_width_(320),
821 codec_height_(240),
Åsa Persson8c1bf952018-09-13 10:42:19 +0200822 max_framerate_(kDefaultFramerate),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200823 fake_encoder_(&time_controller_),
Niels Möller4db138e2018-04-19 09:04:13 +0200824 encoder_factory_(&fake_encoder_),
sprangc5d62e22017-04-02 23:53:04 -0700825 stats_proxy_(new MockableSendStatisticsProxy(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200826 time_controller_.GetClock(),
perkj803d97f2016-11-01 11:45:46 -0700827 video_send_config_,
Jonas Oreland8ca06132022-03-14 12:52:48 +0100828 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
829 field_trials_)),
Tomas Gunnarsson612445e2020-09-21 14:31:23 +0200830 sink_(&time_controller_, &fake_encoder_) {}
perkj26091b12016-09-01 01:17:40 -0700831
832 void SetUp() override {
perkj803d97f2016-11-01 11:45:46 -0700833 metrics::Reset();
perkj26091b12016-09-01 01:17:40 -0700834 video_send_config_ = VideoSendStream::Config(nullptr);
Niels Möller4db138e2018-04-19 09:04:13 +0200835 video_send_config_.encoder_settings.encoder_factory = &encoder_factory_;
Jiawei Ouc2ebe212018-11-08 10:02:56 -0800836 video_send_config_.encoder_settings.bitrate_allocator_factory =
Sergey Silkin5ee69672019-07-02 14:18:34 +0200837 &bitrate_allocator_factory_;
Niels Möller259a4972018-04-05 15:36:51 +0200838 video_send_config_.rtp.payload_name = "FAKE";
839 video_send_config_.rtp.payload_type = 125;
perkj26091b12016-09-01 01:17:40 -0700840
Per512ecb32016-09-23 15:52:06 +0200841 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +0200842 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Åsa Persson17107062020-10-08 08:57:51 +0200843 EXPECT_EQ(1u, video_encoder_config.simulcast_layers.size());
844 video_encoder_config.simulcast_layers[0].num_temporal_layers = 1;
845 video_encoder_config.simulcast_layers[0].max_framerate = max_framerate_;
Erik Språng08127a92016-11-16 16:41:30 +0100846 video_encoder_config_ = video_encoder_config.Copy();
sprang4847ae62017-06-27 07:06:52 -0700847
Niels Möllerf1338562018-04-26 09:51:47 +0200848 ConfigureEncoder(std::move(video_encoder_config));
asapersson5f7226f2016-11-25 04:37:00 -0800849 }
850
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100851 void ConfigureEncoder(
852 VideoEncoderConfig video_encoder_config,
853 VideoStreamEncoder::BitrateAllocationCallbackType
854 allocation_callback_type =
855 VideoStreamEncoder::BitrateAllocationCallbackType::
Erik Språnge4589cb2022-04-06 16:44:30 +0200856 kVideoBitrateAllocationWhenScreenSharing,
857 int num_cores = 1) {
mflodmancc3d4422017-08-03 08:27:51 -0700858 if (video_stream_encoder_)
859 video_stream_encoder_->Stop();
Markus Handell9a478b52021-11-18 16:07:01 +0100860
861 auto encoder_queue = GetTaskQueueFactory()->CreateTaskQueue(
862 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
863 TaskQueueBase* encoder_queue_ptr = encoder_queue.get();
864 std::unique_ptr<FrameCadenceAdapterInterface> cadence_adapter =
865 FrameCadenceAdapterInterface::Create(time_controller_.GetClock(),
Jonas Oreland8ca06132022-03-14 12:52:48 +0100866 encoder_queue_ptr, field_trials_);
Markus Handell9a478b52021-11-18 16:07:01 +0100867 video_stream_encoder_ = std::make_unique<VideoStreamEncoderUnderTest>(
868 &time_controller_, std::move(cadence_adapter), std::move(encoder_queue),
869 stats_proxy_.get(), video_send_config_.encoder_settings,
Erik Språnge4589cb2022-04-06 16:44:30 +0200870 allocation_callback_type, field_trials_, num_cores);
Asa Persson606d3cb2021-10-04 10:07:11 +0200871 video_stream_encoder_->SetSink(&sink_, /*rotation_applied=*/false);
mflodmancc3d4422017-08-03 08:27:51 -0700872 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -0700873 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Asa Persson606d3cb2021-10-04 10:07:11 +0200874 video_stream_encoder_->SetStartBitrate(kTargetBitrate.bps());
mflodmancc3d4422017-08-03 08:27:51 -0700875 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +0200876 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -0700877 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
asapersson5f7226f2016-11-25 04:37:00 -0800878 }
879
Per Kjellanderb03b6c82021-01-03 10:26:03 +0100880 void ResetEncoder(const std::string& payload_name,
881 size_t num_streams,
882 size_t num_temporal_layers,
883 unsigned char num_spatial_layers,
884 bool screenshare,
885 VideoStreamEncoder::BitrateAllocationCallbackType
886 allocation_callback_type =
887 VideoStreamEncoder::BitrateAllocationCallbackType::
Erik Språnge4589cb2022-04-06 16:44:30 +0200888 kVideoBitrateAllocationWhenScreenSharing,
889 int num_cores = 1) {
Niels Möller259a4972018-04-05 15:36:51 +0200890 video_send_config_.rtp.payload_name = payload_name;
asapersson5f7226f2016-11-25 04:37:00 -0800891
892 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +0200893 test::FillEncoderConfiguration(PayloadStringToCodecType(payload_name),
894 num_streams, &video_encoder_config);
895 for (auto& layer : video_encoder_config.simulcast_layers) {
896 layer.num_temporal_layers = num_temporal_layers;
897 layer.max_framerate = kDefaultFramerate;
898 }
Erik Språngd7329ca2019-02-21 21:19:53 +0100899 video_encoder_config.max_bitrate_bps =
Asa Persson606d3cb2021-10-04 10:07:11 +0200900 num_streams == 1 ? kTargetBitrate.bps() : kSimulcastTargetBitrate.bps();
sprang4847ae62017-06-27 07:06:52 -0700901 video_encoder_config.content_type =
902 screenshare ? VideoEncoderConfig::ContentType::kScreen
903 : VideoEncoderConfig::ContentType::kRealtimeVideo;
emircanbbcc3562017-08-18 00:28:40 -0700904 if (payload_name == "VP9") {
905 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
906 vp9_settings.numberOfSpatialLayers = num_spatial_layers;
Mirta Dvornicic97910da2020-07-14 15:29:23 +0200907 vp9_settings.automaticResizeOn = num_spatial_layers <= 1;
emircanbbcc3562017-08-18 00:28:40 -0700908 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200909 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
910 vp9_settings);
emircanbbcc3562017-08-18 00:28:40 -0700911 }
Erik Språnge4589cb2022-04-06 16:44:30 +0200912 ConfigureEncoder(std::move(video_encoder_config), allocation_callback_type,
913 num_cores);
perkj26091b12016-09-01 01:17:40 -0700914 }
915
sprang57c2fff2017-01-16 06:24:02 -0800916 VideoFrame CreateFrame(int64_t ntp_time_ms,
917 rtc::Event* destruction_event) const {
Åsa Persson90719572021-04-08 19:05:30 +0200918 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200919 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200920 destruction_event, codec_width_, codec_height_))
921 .set_ntp_time_ms(ntp_time_ms)
922 .set_timestamp_ms(99)
923 .set_rotation(kVideoRotation_0)
924 .build();
perkj26091b12016-09-01 01:17:40 -0700925 }
926
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100927 VideoFrame CreateFrameWithUpdatedPixel(int64_t ntp_time_ms,
928 rtc::Event* destruction_event,
929 int offset_x) const {
Åsa Persson90719572021-04-08 19:05:30 +0200930 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200931 .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200932 destruction_event, codec_width_, codec_height_))
933 .set_ntp_time_ms(ntp_time_ms)
934 .set_timestamp_ms(99)
935 .set_rotation(kVideoRotation_0)
936 .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
937 .build();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +0100938 }
939
sprang57c2fff2017-01-16 06:24:02 -0800940 VideoFrame CreateFrame(int64_t ntp_time_ms, int width, int height) const {
Erik Språng7444b192021-06-02 14:02:13 +0200941 auto buffer = rtc::make_ref_counted<TestBuffer>(nullptr, width, height);
942 I420Buffer::SetBlack(buffer.get());
Åsa Persson90719572021-04-08 19:05:30 +0200943 return VideoFrame::Builder()
Erik Språng7444b192021-06-02 14:02:13 +0200944 .set_video_frame_buffer(std::move(buffer))
Åsa Persson90719572021-04-08 19:05:30 +0200945 .set_ntp_time_ms(ntp_time_ms)
946 .set_timestamp_ms(ntp_time_ms)
947 .set_rotation(kVideoRotation_0)
948 .build();
perkj803d97f2016-11-01 11:45:46 -0700949 }
950
Evan Shrubsole895556e2020-10-05 09:15:13 +0200951 VideoFrame CreateNV12Frame(int64_t ntp_time_ms, int width, int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200952 return VideoFrame::Builder()
953 .set_video_frame_buffer(NV12Buffer::Create(width, height))
954 .set_ntp_time_ms(ntp_time_ms)
955 .set_timestamp_ms(ntp_time_ms)
956 .set_rotation(kVideoRotation_0)
957 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200958 }
959
Noah Richards51db4212019-06-12 06:59:12 -0700960 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
961 rtc::Event* destruction_event,
962 int width,
963 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200964 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200965 .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200966 destruction_event, width, height))
967 .set_ntp_time_ms(ntp_time_ms)
968 .set_timestamp_ms(99)
969 .set_rotation(kVideoRotation_0)
970 .build();
Noah Richards51db4212019-06-12 06:59:12 -0700971 }
972
Evan Shrubsole895556e2020-10-05 09:15:13 +0200973 VideoFrame CreateFakeNV12NativeFrame(int64_t ntp_time_ms,
974 rtc::Event* destruction_event,
975 int width,
976 int height) const {
Åsa Persson90719572021-04-08 19:05:30 +0200977 return VideoFrame::Builder()
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +0200978 .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
Åsa Persson90719572021-04-08 19:05:30 +0200979 destruction_event, width, height))
980 .set_ntp_time_ms(ntp_time_ms)
981 .set_timestamp_ms(99)
982 .set_rotation(kVideoRotation_0)
983 .build();
Evan Shrubsole895556e2020-10-05 09:15:13 +0200984 }
985
Noah Richards51db4212019-06-12 06:59:12 -0700986 VideoFrame CreateFakeNativeFrame(int64_t ntp_time_ms,
987 rtc::Event* destruction_event) const {
988 return CreateFakeNativeFrame(ntp_time_ms, destruction_event, codec_width_,
989 codec_height_);
990 }
991
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100992 void VerifyAllocatedBitrate(const VideoBitrateAllocation& expected_bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +0200993 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +0200994 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +0100995
996 video_source_.IncomingCapturedFrame(
997 CreateFrame(1, codec_width_, codec_height_));
998 WaitForEncodedFrame(1);
Per Kjellanderdcef6412020-10-07 15:09:05 +0200999 EXPECT_EQ(expected_bitrate, sink_.GetLastVideoBitrateAllocation());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001000 }
1001
sprang4847ae62017-06-27 07:06:52 -07001002 void WaitForEncodedFrame(int64_t expected_ntp_time) {
1003 sink_.WaitForEncodedFrame(expected_ntp_time);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001004 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001005 }
1006
1007 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time, int64_t timeout_ms) {
1008 bool ok = sink_.TimedWaitForEncodedFrame(expected_ntp_time, timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001009 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001010 return ok;
1011 }
1012
1013 void WaitForEncodedFrame(uint32_t expected_width, uint32_t expected_height) {
1014 sink_.WaitForEncodedFrame(expected_width, expected_height);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001015 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001016 }
1017
1018 void ExpectDroppedFrame() {
1019 sink_.ExpectDroppedFrame();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001020 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001021 }
1022
1023 bool WaitForFrame(int64_t timeout_ms) {
1024 bool ok = sink_.WaitForFrame(timeout_ms);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001025 AdvanceTime(TimeDelta::Seconds(1) / max_framerate_);
sprang4847ae62017-06-27 07:06:52 -07001026 return ok;
1027 }
1028
perkj26091b12016-09-01 01:17:40 -07001029 class TestEncoder : public test::FakeEncoder {
1030 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001031 explicit TestEncoder(TimeController* time_controller)
1032 : FakeEncoder(time_controller->GetClock()),
1033 time_controller_(time_controller) {
1034 RTC_DCHECK(time_controller_);
1035 }
perkj26091b12016-09-01 01:17:40 -07001036
Erik Språngaed30702018-11-05 12:57:17 +01001037 VideoEncoder::EncoderInfo GetEncoderInfo() const override {
Markus Handella3765182020-07-08 13:13:32 +02001038 MutexLock lock(&local_mutex_);
Erik Språng9d69cbe2020-10-22 17:44:42 +02001039 EncoderInfo info = FakeEncoder::GetEncoderInfo();
Erik Språngb7cb7b52019-02-26 15:52:33 +01001040 if (initialized_ == EncoderState::kInitialized) {
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001041 if (quality_scaling_) {
Åsa Perssone644a032019-11-08 15:56:00 +01001042 info.scaling_settings = VideoEncoder::ScalingSettings(
1043 kQpLow, kQpHigh, kMinPixelsPerFrame);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001044 }
1045 info.is_hardware_accelerated = is_hardware_accelerated_;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001046 for (int i = 0; i < kMaxSpatialLayers; ++i) {
1047 if (temporal_layers_supported_[i]) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001048 info.fps_allocation[i].clear();
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001049 int num_layers = temporal_layers_supported_[i].value() ? 2 : 1;
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01001050 for (int tid = 0; tid < num_layers; ++tid)
1051 info.fps_allocation[i].push_back(255 / (num_layers - tid));
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001052 }
1053 }
Erik Språngaed30702018-11-05 12:57:17 +01001054 }
Sergey Silkin6456e352019-07-08 17:56:40 +02001055
1056 info.resolution_bitrate_limits = resolution_bitrate_limits_;
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001057 info.requested_resolution_alignment = requested_resolution_alignment_;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001058 info.apply_alignment_to_all_simulcast_layers =
1059 apply_alignment_to_all_simulcast_layers_;
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001060 info.preferred_pixel_formats = preferred_pixel_formats_;
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001061 if (is_qp_trusted_.has_value()) {
1062 info.is_qp_trusted = is_qp_trusted_;
1063 }
Erik Språngaed30702018-11-05 12:57:17 +01001064 return info;
kthelgason876222f2016-11-29 01:44:11 -08001065 }
1066
Erik Språngb7cb7b52019-02-26 15:52:33 +01001067 int32_t RegisterEncodeCompleteCallback(
1068 EncodedImageCallback* callback) override {
Markus Handella3765182020-07-08 13:13:32 +02001069 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001070 encoded_image_callback_ = callback;
1071 return FakeEncoder::RegisterEncodeCompleteCallback(callback);
1072 }
1073
perkjfa10b552016-10-02 23:45:26 -07001074 void ContinueEncode() { continue_encode_event_.Set(); }
1075
1076 void CheckLastTimeStampsMatch(int64_t ntp_time_ms,
1077 uint32_t timestamp) const {
Markus Handella3765182020-07-08 13:13:32 +02001078 MutexLock lock(&local_mutex_);
perkjfa10b552016-10-02 23:45:26 -07001079 EXPECT_EQ(timestamp_, timestamp);
1080 EXPECT_EQ(ntp_time_ms_, ntp_time_ms);
1081 }
1082
kthelgason2fc52542017-03-03 00:24:41 -08001083 void SetQualityScaling(bool b) {
Markus Handella3765182020-07-08 13:13:32 +02001084 MutexLock lock(&local_mutex_);
kthelgason2fc52542017-03-03 00:24:41 -08001085 quality_scaling_ = b;
1086 }
kthelgasonad9010c2017-02-14 00:46:51 -08001087
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001088 void SetRequestedResolutionAlignment(int requested_resolution_alignment) {
Markus Handella3765182020-07-08 13:13:32 +02001089 MutexLock lock(&local_mutex_);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01001090 requested_resolution_alignment_ = requested_resolution_alignment;
1091 }
1092
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001093 void SetApplyAlignmentToAllSimulcastLayers(bool b) {
1094 MutexLock lock(&local_mutex_);
1095 apply_alignment_to_all_simulcast_layers_ = b;
1096 }
1097
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001098 void SetIsHardwareAccelerated(bool is_hardware_accelerated) {
Markus Handella3765182020-07-08 13:13:32 +02001099 MutexLock lock(&local_mutex_);
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001100 is_hardware_accelerated_ = is_hardware_accelerated;
1101 }
1102
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001103 void SetTemporalLayersSupported(size_t spatial_idx, bool supported) {
1104 RTC_DCHECK_LT(spatial_idx, kMaxSpatialLayers);
Markus Handella3765182020-07-08 13:13:32 +02001105 MutexLock lock(&local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001106 temporal_layers_supported_[spatial_idx] = supported;
1107 }
1108
Sergey Silkin6456e352019-07-08 17:56:40 +02001109 void SetResolutionBitrateLimits(
1110 std::vector<ResolutionBitrateLimits> thresholds) {
Markus Handella3765182020-07-08 13:13:32 +02001111 MutexLock lock(&local_mutex_);
Sergey Silkin6456e352019-07-08 17:56:40 +02001112 resolution_bitrate_limits_ = thresholds;
1113 }
1114
sprangfe627f32017-03-29 08:24:59 -07001115 void ForceInitEncodeFailure(bool force_failure) {
Markus Handella3765182020-07-08 13:13:32 +02001116 MutexLock lock(&local_mutex_);
sprangfe627f32017-03-29 08:24:59 -07001117 force_init_encode_failed_ = force_failure;
1118 }
1119
Niels Möller6bb5ab92019-01-11 11:11:10 +01001120 void SimulateOvershoot(double rate_factor) {
Markus Handella3765182020-07-08 13:13:32 +02001121 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001122 rate_factor_ = rate_factor;
1123 }
1124
Erik Språngd7329ca2019-02-21 21:19:53 +01001125 uint32_t GetLastFramerate() const {
Markus Handella3765182020-07-08 13:13:32 +02001126 MutexLock lock(&local_mutex_);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001127 return last_framerate_;
1128 }
1129
Erik Språngd7329ca2019-02-21 21:19:53 +01001130 VideoFrame::UpdateRect GetLastUpdateRect() const {
Markus Handella3765182020-07-08 13:13:32 +02001131 MutexLock lock(&local_mutex_);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001132 return last_update_rect_;
1133 }
1134
Niels Möller87e2d782019-03-07 10:18:23 +01001135 const std::vector<VideoFrameType>& LastFrameTypes() const {
Markus Handella3765182020-07-08 13:13:32 +02001136 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001137 return last_frame_types_;
1138 }
1139
1140 void InjectFrame(const VideoFrame& input_image, bool keyframe) {
Niels Möller87e2d782019-03-07 10:18:23 +01001141 const std::vector<VideoFrameType> frame_type = {
Niels Möller8f7ce222019-03-21 15:43:58 +01001142 keyframe ? VideoFrameType::kVideoFrameKey
1143 : VideoFrameType::kVideoFrameDelta};
Erik Språngd7329ca2019-02-21 21:19:53 +01001144 {
Markus Handella3765182020-07-08 13:13:32 +02001145 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001146 last_frame_types_ = frame_type;
1147 }
Niels Möllerb859b322019-03-07 12:40:01 +01001148 FakeEncoder::Encode(input_image, &frame_type);
Erik Språngd7329ca2019-02-21 21:19:53 +01001149 }
1150
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001151 void InjectEncodedImage(const EncodedImage& image,
1152 const CodecSpecificInfo* codec_specific_info) {
Markus Handella3765182020-07-08 13:13:32 +02001153 MutexLock lock(&local_mutex_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001154 encoded_image_callback_->OnEncodedImage(image, codec_specific_info);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001155 }
1156
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001157 void SetEncodedImageData(
1158 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data) {
Markus Handella3765182020-07-08 13:13:32 +02001159 MutexLock lock(&local_mutex_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001160 encoded_image_data_ = encoded_image_data;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001161 }
1162
Erik Språngd7329ca2019-02-21 21:19:53 +01001163 void ExpectNullFrame() {
Markus Handella3765182020-07-08 13:13:32 +02001164 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001165 expect_null_frame_ = true;
1166 }
1167
Erik Språng5056af02019-09-02 15:53:11 +02001168 absl::optional<VideoEncoder::RateControlParameters>
1169 GetAndResetLastRateControlSettings() {
1170 auto settings = last_rate_control_settings_;
1171 last_rate_control_settings_.reset();
1172 return settings;
Erik Språngd7329ca2019-02-21 21:19:53 +01001173 }
1174
Henrik Boström56db9ff2021-03-24 09:06:45 +01001175 int GetLastInputWidth() const {
1176 MutexLock lock(&local_mutex_);
1177 return last_input_width_;
1178 }
1179
1180 int GetLastInputHeight() const {
1181 MutexLock lock(&local_mutex_);
1182 return last_input_height_;
1183 }
1184
Evan Shrubsole895556e2020-10-05 09:15:13 +02001185 absl::optional<VideoFrameBuffer::Type> GetLastInputPixelFormat() {
1186 MutexLock lock(&local_mutex_);
1187 return last_input_pixel_format_;
1188 }
1189
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001190 int GetNumSetRates() const {
Markus Handella3765182020-07-08 13:13:32 +02001191 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001192 return num_set_rates_;
1193 }
1194
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001195 void SetPreferredPixelFormats(
1196 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1197 pixel_formats) {
1198 MutexLock lock(&local_mutex_);
1199 preferred_pixel_formats_ = std::move(pixel_formats);
1200 }
1201
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001202 void SetIsQpTrusted(absl::optional<bool> trusted) {
1203 MutexLock lock(&local_mutex_);
1204 is_qp_trusted_ = trusted;
1205 }
1206
Erik Språnge4589cb2022-04-06 16:44:30 +02001207 VideoCodecComplexity LastEncoderComplexity() {
1208 MutexLock lock(&local_mutex_);
1209 return last_encoder_complexity_;
1210 }
1211
perkjfa10b552016-10-02 23:45:26 -07001212 private:
perkj26091b12016-09-01 01:17:40 -07001213 int32_t Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +01001214 const std::vector<VideoFrameType>* frame_types) override {
perkj26091b12016-09-01 01:17:40 -07001215 {
Markus Handella3765182020-07-08 13:13:32 +02001216 MutexLock lock(&local_mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001217 if (expect_null_frame_) {
1218 EXPECT_EQ(input_image.timestamp(), 0u);
1219 EXPECT_EQ(input_image.width(), 1);
1220 last_frame_types_ = *frame_types;
1221 expect_null_frame_ = false;
1222 } else {
1223 EXPECT_GT(input_image.timestamp(), timestamp_);
1224 EXPECT_GT(input_image.ntp_time_ms(), ntp_time_ms_);
1225 EXPECT_EQ(input_image.timestamp(), input_image.ntp_time_ms() * 90);
1226 }
perkj26091b12016-09-01 01:17:40 -07001227
1228 timestamp_ = input_image.timestamp();
1229 ntp_time_ms_ = input_image.ntp_time_ms();
perkj803d97f2016-11-01 11:45:46 -07001230 last_input_width_ = input_image.width();
1231 last_input_height_ = input_image.height();
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01001232 last_update_rect_ = input_image.update_rect();
Erik Språngd7329ca2019-02-21 21:19:53 +01001233 last_frame_types_ = *frame_types;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001234 last_input_pixel_format_ = input_image.video_frame_buffer()->type();
perkj26091b12016-09-01 01:17:40 -07001235 }
Niels Möllerb859b322019-03-07 12:40:01 +01001236 int32_t result = FakeEncoder::Encode(input_image, frame_types);
perkj26091b12016-09-01 01:17:40 -07001237 return result;
1238 }
1239
Niels Möller08ae7ce2020-09-23 15:58:12 +02001240 CodecSpecificInfo EncodeHook(
1241 EncodedImage& encoded_image,
1242 rtc::scoped_refptr<EncodedImageBuffer> buffer) override {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001243 CodecSpecificInfo codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001244 {
1245 MutexLock lock(&mutex_);
Danil Chapovalov2549f172020-08-12 17:30:36 +02001246 codec_specific.codecType = config_.codecType;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001247 }
1248 MutexLock lock(&local_mutex_);
1249 if (encoded_image_data_) {
Danil Chapovalov2549f172020-08-12 17:30:36 +02001250 encoded_image.SetEncodedData(encoded_image_data_);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001251 }
Danil Chapovalov2549f172020-08-12 17:30:36 +02001252 return codec_specific;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001253 }
1254
sprangfe627f32017-03-29 08:24:59 -07001255 int32_t InitEncode(const VideoCodec* config,
Elad Alon370f93a2019-06-11 14:57:57 +02001256 const Settings& settings) override {
1257 int res = FakeEncoder::InitEncode(config, settings);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001258
Markus Handella3765182020-07-08 13:13:32 +02001259 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001260 EXPECT_EQ(initialized_, EncoderState::kUninitialized);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001261
Erik Språng82fad3d2018-03-21 09:57:23 +01001262 if (config->codecType == kVideoCodecVP8) {
sprangfe627f32017-03-29 08:24:59 -07001263 // Simulate setting up temporal layers, in order to validate the life
1264 // cycle of these objects.
Elad Aloncde8ab22019-03-20 11:56:20 +01001265 Vp8TemporalLayersFactory factory;
Elad Alon45befc52019-07-02 11:20:09 +02001266 frame_buffer_controller_ =
1267 factory.Create(*config, settings, &fec_controller_override_);
sprangfe627f32017-03-29 08:24:59 -07001268 }
Erik Språnge4589cb2022-04-06 16:44:30 +02001269
1270 last_encoder_complexity_ = config->GetVideoEncoderComplexity();
1271
Erik Språngb7cb7b52019-02-26 15:52:33 +01001272 if (force_init_encode_failed_) {
1273 initialized_ = EncoderState::kInitializationFailed;
sprangfe627f32017-03-29 08:24:59 -07001274 return -1;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001275 }
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01001276
Erik Språngb7cb7b52019-02-26 15:52:33 +01001277 initialized_ = EncoderState::kInitialized;
sprangfe627f32017-03-29 08:24:59 -07001278 return res;
1279 }
1280
Erik Språngb7cb7b52019-02-26 15:52:33 +01001281 int32_t Release() override {
Markus Handella3765182020-07-08 13:13:32 +02001282 MutexLock lock(&local_mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001283 EXPECT_NE(initialized_, EncoderState::kUninitialized);
1284 initialized_ = EncoderState::kUninitialized;
1285 return FakeEncoder::Release();
1286 }
1287
Erik Språng16cb8f52019-04-12 13:59:09 +02001288 void SetRates(const RateControlParameters& parameters) {
Markus Handella3765182020-07-08 13:13:32 +02001289 MutexLock lock(&local_mutex_);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02001290 num_set_rates_++;
Niels Möller6bb5ab92019-01-11 11:11:10 +01001291 VideoBitrateAllocation adjusted_rate_allocation;
1292 for (size_t si = 0; si < kMaxSpatialLayers; ++si) {
1293 for (size_t ti = 0; ti < kMaxTemporalStreams; ++ti) {
Erik Språng16cb8f52019-04-12 13:59:09 +02001294 if (parameters.bitrate.HasBitrate(si, ti)) {
Niels Möller6bb5ab92019-01-11 11:11:10 +01001295 adjusted_rate_allocation.SetBitrate(
1296 si, ti,
Erik Språng16cb8f52019-04-12 13:59:09 +02001297 static_cast<uint32_t>(parameters.bitrate.GetBitrate(si, ti) *
Niels Möller6bb5ab92019-01-11 11:11:10 +01001298 rate_factor_));
1299 }
1300 }
1301 }
Erik Språng16cb8f52019-04-12 13:59:09 +02001302 last_framerate_ = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
Erik Språng5056af02019-09-02 15:53:11 +02001303 last_rate_control_settings_ = parameters;
Erik Språng16cb8f52019-04-12 13:59:09 +02001304 RateControlParameters adjusted_paramters = parameters;
1305 adjusted_paramters.bitrate = adjusted_rate_allocation;
1306 FakeEncoder::SetRates(adjusted_paramters);
Niels Möller6bb5ab92019-01-11 11:11:10 +01001307 }
1308
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001309 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001310 mutable Mutex local_mutex_;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001311 enum class EncoderState {
1312 kUninitialized,
1313 kInitializationFailed,
1314 kInitialized
Markus Handella3765182020-07-08 13:13:32 +02001315 } initialized_ RTC_GUARDED_BY(local_mutex_) = EncoderState::kUninitialized;
perkj26091b12016-09-01 01:17:40 -07001316 rtc::Event continue_encode_event_;
Markus Handella3765182020-07-08 13:13:32 +02001317 uint32_t timestamp_ RTC_GUARDED_BY(local_mutex_) = 0;
1318 int64_t ntp_time_ms_ RTC_GUARDED_BY(local_mutex_) = 0;
1319 int last_input_width_ RTC_GUARDED_BY(local_mutex_) = 0;
1320 int last_input_height_ RTC_GUARDED_BY(local_mutex_) = 0;
1321 bool quality_scaling_ RTC_GUARDED_BY(local_mutex_) = true;
1322 int requested_resolution_alignment_ RTC_GUARDED_BY(local_mutex_) = 1;
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02001323 bool apply_alignment_to_all_simulcast_layers_ RTC_GUARDED_BY(local_mutex_) =
1324 false;
Markus Handella3765182020-07-08 13:13:32 +02001325 bool is_hardware_accelerated_ RTC_GUARDED_BY(local_mutex_) = false;
Mirta Dvornicic97910da2020-07-14 15:29:23 +02001326 rtc::scoped_refptr<EncodedImageBufferInterface> encoded_image_data_
1327 RTC_GUARDED_BY(local_mutex_);
Elad Aloncde8ab22019-03-20 11:56:20 +01001328 std::unique_ptr<Vp8FrameBufferController> frame_buffer_controller_
Markus Handella3765182020-07-08 13:13:32 +02001329 RTC_GUARDED_BY(local_mutex_);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01001330 absl::optional<bool>
1331 temporal_layers_supported_[kMaxSpatialLayers] RTC_GUARDED_BY(
Markus Handella3765182020-07-08 13:13:32 +02001332 local_mutex_);
1333 bool force_init_encode_failed_ RTC_GUARDED_BY(local_mutex_) = false;
1334 double rate_factor_ RTC_GUARDED_BY(local_mutex_) = 1.0;
1335 uint32_t last_framerate_ RTC_GUARDED_BY(local_mutex_) = 0;
Erik Språng5056af02019-09-02 15:53:11 +02001336 absl::optional<VideoEncoder::RateControlParameters>
1337 last_rate_control_settings_;
Markus Handella3765182020-07-08 13:13:32 +02001338 VideoFrame::UpdateRect last_update_rect_ RTC_GUARDED_BY(local_mutex_) = {
1339 0, 0, 0, 0};
Niels Möller87e2d782019-03-07 10:18:23 +01001340 std::vector<VideoFrameType> last_frame_types_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001341 bool expect_null_frame_ = false;
Markus Handella3765182020-07-08 13:13:32 +02001342 EncodedImageCallback* encoded_image_callback_ RTC_GUARDED_BY(local_mutex_) =
1343 nullptr;
Evan Shrubsolefb862742020-03-16 16:18:36 +01001344 NiceMock<MockFecControllerOverride> fec_controller_override_;
Sergey Silkin6456e352019-07-08 17:56:40 +02001345 std::vector<ResolutionBitrateLimits> resolution_bitrate_limits_
Markus Handella3765182020-07-08 13:13:32 +02001346 RTC_GUARDED_BY(local_mutex_);
1347 int num_set_rates_ RTC_GUARDED_BY(local_mutex_) = 0;
Evan Shrubsole895556e2020-10-05 09:15:13 +02001348 absl::optional<VideoFrameBuffer::Type> last_input_pixel_format_
1349 RTC_GUARDED_BY(local_mutex_);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001350 absl::InlinedVector<VideoFrameBuffer::Type, kMaxPreferredPixelFormats>
1351 preferred_pixel_formats_ RTC_GUARDED_BY(local_mutex_);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08001352 absl::optional<bool> is_qp_trusted_ RTC_GUARDED_BY(local_mutex_);
Erik Språnge4589cb2022-04-06 16:44:30 +02001353 VideoCodecComplexity last_encoder_complexity_ RTC_GUARDED_BY(local_mutex_){
1354 VideoCodecComplexity::kComplexityNormal};
perkj26091b12016-09-01 01:17:40 -07001355 };
1356
mflodmancc3d4422017-08-03 08:27:51 -07001357 class TestSink : public VideoStreamEncoder::EncoderSink {
perkj26091b12016-09-01 01:17:40 -07001358 public:
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001359 TestSink(TimeController* time_controller, TestEncoder* test_encoder)
1360 : time_controller_(time_controller), test_encoder_(test_encoder) {
1361 RTC_DCHECK(time_controller_);
1362 }
perkj26091b12016-09-01 01:17:40 -07001363
perkj26091b12016-09-01 01:17:40 -07001364 void WaitForEncodedFrame(int64_t expected_ntp_time) {
sprang4847ae62017-06-27 07:06:52 -07001365 EXPECT_TRUE(
1366 TimedWaitForEncodedFrame(expected_ntp_time, kDefaultTimeoutMs));
1367 }
1368
1369 bool TimedWaitForEncodedFrame(int64_t expected_ntp_time,
1370 int64_t timeout_ms) {
perkj26091b12016-09-01 01:17:40 -07001371 uint32_t timestamp = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001372 if (!WaitForFrame(timeout_ms))
sprang4847ae62017-06-27 07:06:52 -07001373 return false;
perkj26091b12016-09-01 01:17:40 -07001374 {
Markus Handella3765182020-07-08 13:13:32 +02001375 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001376 timestamp = last_timestamp_;
perkj26091b12016-09-01 01:17:40 -07001377 }
1378 test_encoder_->CheckLastTimeStampsMatch(expected_ntp_time, timestamp);
sprang4847ae62017-06-27 07:06:52 -07001379 return true;
perkj26091b12016-09-01 01:17:40 -07001380 }
1381
sprangb1ca0732017-02-01 08:38:12 -08001382 void WaitForEncodedFrame(uint32_t expected_width,
1383 uint32_t expected_height) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001384 EXPECT_TRUE(WaitForFrame(kDefaultTimeoutMs));
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001385 CheckLastFrameSizeMatches(expected_width, expected_height);
sprangc5d62e22017-04-02 23:53:04 -07001386 }
1387
Åsa Perssonc74d8da2017-12-04 14:13:56 +01001388 void CheckLastFrameSizeMatches(uint32_t expected_width,
sprangc5d62e22017-04-02 23:53:04 -07001389 uint32_t expected_height) {
sprangb1ca0732017-02-01 08:38:12 -08001390 uint32_t width = 0;
1391 uint32_t height = 0;
sprangb1ca0732017-02-01 08:38:12 -08001392 {
Markus Handella3765182020-07-08 13:13:32 +02001393 MutexLock lock(&mutex_);
sprangb1ca0732017-02-01 08:38:12 -08001394 width = last_width_;
1395 height = last_height_;
1396 }
1397 EXPECT_EQ(expected_height, height);
1398 EXPECT_EQ(expected_width, width);
1399 }
1400
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001401 void CheckLastFrameRotationMatches(VideoRotation expected_rotation) {
1402 VideoRotation rotation;
1403 {
Markus Handella3765182020-07-08 13:13:32 +02001404 MutexLock lock(&mutex_);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001405 rotation = last_rotation_;
1406 }
1407 EXPECT_EQ(expected_rotation, rotation);
1408 }
1409
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001410 void ExpectDroppedFrame() { EXPECT_FALSE(WaitForFrame(100)); }
kthelgason2bc68642017-02-07 07:02:22 -08001411
sprangc5d62e22017-04-02 23:53:04 -07001412 bool WaitForFrame(int64_t timeout_ms) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001413 RTC_DCHECK(time_controller_->GetMainThread()->IsCurrent());
Markus Handell28c71802021-11-08 10:11:55 +01001414 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001415 bool ret = encoded_frame_event_.Wait(timeout_ms);
Markus Handell28c71802021-11-08 10:11:55 +01001416 time_controller_->AdvanceTime(TimeDelta::Zero());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001417 return ret;
sprangc5d62e22017-04-02 23:53:04 -07001418 }
1419
perkj26091b12016-09-01 01:17:40 -07001420 void SetExpectNoFrames() {
Markus Handella3765182020-07-08 13:13:32 +02001421 MutexLock lock(&mutex_);
perkj26091b12016-09-01 01:17:40 -07001422 expect_frames_ = false;
1423 }
1424
asaperssonfab67072017-04-04 05:51:49 -07001425 int number_of_reconfigurations() const {
Markus Handella3765182020-07-08 13:13:32 +02001426 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001427 return number_of_reconfigurations_;
1428 }
1429
asaperssonfab67072017-04-04 05:51:49 -07001430 int last_min_transmit_bitrate() const {
Markus Handella3765182020-07-08 13:13:32 +02001431 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001432 return min_transmit_bitrate_bps_;
1433 }
1434
Erik Språngd7329ca2019-02-21 21:19:53 +01001435 void SetNumExpectedLayers(size_t num_layers) {
Markus Handella3765182020-07-08 13:13:32 +02001436 MutexLock lock(&mutex_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001437 num_expected_layers_ = num_layers;
1438 }
1439
Erik Språngb7cb7b52019-02-26 15:52:33 +01001440 int64_t GetLastCaptureTimeMs() const {
Markus Handella3765182020-07-08 13:13:32 +02001441 MutexLock lock(&mutex_);
Erik Språngb7cb7b52019-02-26 15:52:33 +01001442 return last_capture_time_ms_;
1443 }
1444
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001445 const EncodedImage& GetLastEncodedImage() {
1446 MutexLock lock(&mutex_);
1447 return last_encoded_image_;
1448 }
1449
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001450 std::vector<uint8_t> GetLastEncodedImageData() {
Markus Handella3765182020-07-08 13:13:32 +02001451 MutexLock lock(&mutex_);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001452 return std::move(last_encoded_image_data_);
1453 }
1454
Per Kjellanderdcef6412020-10-07 15:09:05 +02001455 VideoBitrateAllocation GetLastVideoBitrateAllocation() {
1456 MutexLock lock(&mutex_);
1457 return last_bitrate_allocation_;
1458 }
1459
1460 int number_of_bitrate_allocations() const {
1461 MutexLock lock(&mutex_);
1462 return number_of_bitrate_allocations_;
1463 }
1464
Per Kjellandera9434842020-10-15 17:53:22 +02001465 VideoLayersAllocation GetLastVideoLayersAllocation() {
1466 MutexLock lock(&mutex_);
1467 return last_layers_allocation_;
1468 }
1469
1470 int number_of_layers_allocations() const {
1471 MutexLock lock(&mutex_);
1472 return number_of_layers_allocations_;
1473 }
1474
perkj26091b12016-09-01 01:17:40 -07001475 private:
sergeyu2cb155a2016-11-04 11:39:29 -07001476 Result OnEncodedImage(
1477 const EncodedImage& encoded_image,
Danil Chapovalov2549f172020-08-12 17:30:36 +02001478 const CodecSpecificInfo* codec_specific_info) override {
Markus Handella3765182020-07-08 13:13:32 +02001479 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001480 EXPECT_TRUE(expect_frames_);
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001481 last_encoded_image_ = EncodedImage(encoded_image);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001482 last_encoded_image_data_ = std::vector<uint8_t>(
1483 encoded_image.data(), encoded_image.data() + encoded_image.size());
Erik Språngd7329ca2019-02-21 21:19:53 +01001484 uint32_t timestamp = encoded_image.Timestamp();
1485 if (last_timestamp_ != timestamp) {
1486 num_received_layers_ = 1;
Erik Språng7444b192021-06-02 14:02:13 +02001487 last_width_ = encoded_image._encodedWidth;
1488 last_height_ = encoded_image._encodedHeight;
Erik Språngd7329ca2019-02-21 21:19:53 +01001489 } else {
1490 ++num_received_layers_;
Erik Språng7444b192021-06-02 14:02:13 +02001491 last_width_ = std::max(encoded_image._encodedWidth, last_width_);
1492 last_height_ = std::max(encoded_image._encodedHeight, last_height_);
Erik Språngd7329ca2019-02-21 21:19:53 +01001493 }
1494 last_timestamp_ = timestamp;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001495 last_capture_time_ms_ = encoded_image.capture_time_ms_;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001496 last_rotation_ = encoded_image.rotation_;
Erik Språngd7329ca2019-02-21 21:19:53 +01001497 if (num_received_layers_ == num_expected_layers_) {
1498 encoded_frame_event_.Set();
1499 }
sprangb1ca0732017-02-01 08:38:12 -08001500 return Result(Result::OK, last_timestamp_);
Per512ecb32016-09-23 15:52:06 +02001501 }
1502
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001503 void OnEncoderConfigurationChanged(
1504 std::vector<VideoStream> streams,
Ilya Nikolaevskiy93be66c2020-04-02 14:10:27 +02001505 bool is_svc,
Rasmus Brandtc402dbe2019-02-04 11:09:46 +01001506 VideoEncoderConfig::ContentType content_type,
1507 int min_transmit_bitrate_bps) override {
Markus Handella3765182020-07-08 13:13:32 +02001508 MutexLock lock(&mutex_);
Per512ecb32016-09-23 15:52:06 +02001509 ++number_of_reconfigurations_;
1510 min_transmit_bitrate_bps_ = min_transmit_bitrate_bps;
1511 }
1512
Per Kjellanderdcef6412020-10-07 15:09:05 +02001513 void OnBitrateAllocationUpdated(
1514 const VideoBitrateAllocation& allocation) override {
1515 MutexLock lock(&mutex_);
1516 ++number_of_bitrate_allocations_;
1517 last_bitrate_allocation_ = allocation;
1518 }
1519
Per Kjellandera9434842020-10-15 17:53:22 +02001520 void OnVideoLayersAllocationUpdated(
1521 VideoLayersAllocation allocation) override {
1522 MutexLock lock(&mutex_);
1523 ++number_of_layers_allocations_;
1524 last_layers_allocation_ = allocation;
1525 rtc::StringBuilder log;
1526 for (const auto& layer : allocation.active_spatial_layers) {
1527 log << layer.width << "x" << layer.height << "@" << layer.frame_rate_fps
1528 << "[";
1529 for (const auto target_bitrate :
1530 layer.target_bitrate_per_temporal_layer) {
1531 log << target_bitrate.kbps() << ",";
1532 }
1533 log << "]";
1534 }
Harald Alvestrand97597c02021-11-04 12:01:23 +00001535 RTC_DLOG(LS_INFO) << "OnVideoLayersAllocationUpdated " << log.str();
Per Kjellandera9434842020-10-15 17:53:22 +02001536 }
1537
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001538 TimeController* const time_controller_;
Markus Handella3765182020-07-08 13:13:32 +02001539 mutable Mutex mutex_;
perkj26091b12016-09-01 01:17:40 -07001540 TestEncoder* test_encoder_;
1541 rtc::Event encoded_frame_event_;
Sergey Silkin0e42cf72021-03-15 10:12:57 +01001542 EncodedImage last_encoded_image_;
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02001543 std::vector<uint8_t> last_encoded_image_data_;
sprangb1ca0732017-02-01 08:38:12 -08001544 uint32_t last_timestamp_ = 0;
Erik Språngb7cb7b52019-02-26 15:52:33 +01001545 int64_t last_capture_time_ms_ = 0;
sprangb1ca0732017-02-01 08:38:12 -08001546 uint32_t last_height_ = 0;
1547 uint32_t last_width_ = 0;
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02001548 VideoRotation last_rotation_ = kVideoRotation_0;
Erik Språngd7329ca2019-02-21 21:19:53 +01001549 size_t num_expected_layers_ = 1;
1550 size_t num_received_layers_ = 0;
perkj26091b12016-09-01 01:17:40 -07001551 bool expect_frames_ = true;
Per512ecb32016-09-23 15:52:06 +02001552 int number_of_reconfigurations_ = 0;
1553 int min_transmit_bitrate_bps_ = 0;
Per Kjellanderdcef6412020-10-07 15:09:05 +02001554 VideoBitrateAllocation last_bitrate_allocation_ RTC_GUARDED_BY(&mutex_);
1555 int number_of_bitrate_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
Per Kjellandera9434842020-10-15 17:53:22 +02001556 VideoLayersAllocation last_layers_allocation_ RTC_GUARDED_BY(&mutex_);
1557 int number_of_layers_allocations_ RTC_GUARDED_BY(&mutex_) = 0;
perkj26091b12016-09-01 01:17:40 -07001558 };
1559
Sergey Silkin5ee69672019-07-02 14:18:34 +02001560 class VideoBitrateAllocatorProxyFactory
1561 : public VideoBitrateAllocatorFactory {
1562 public:
1563 VideoBitrateAllocatorProxyFactory()
1564 : bitrate_allocator_factory_(
1565 CreateBuiltinVideoBitrateAllocatorFactory()) {}
1566
1567 std::unique_ptr<VideoBitrateAllocator> CreateVideoBitrateAllocator(
1568 const VideoCodec& codec) override {
Markus Handella3765182020-07-08 13:13:32 +02001569 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001570 codec_config_ = codec;
1571 return bitrate_allocator_factory_->CreateVideoBitrateAllocator(codec);
1572 }
1573
1574 VideoCodec codec_config() const {
Markus Handella3765182020-07-08 13:13:32 +02001575 MutexLock lock(&mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001576 return codec_config_;
1577 }
1578
1579 private:
1580 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory_;
1581
Markus Handella3765182020-07-08 13:13:32 +02001582 mutable Mutex mutex_;
1583 VideoCodec codec_config_ RTC_GUARDED_BY(mutex_);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001584 };
1585
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001586 Clock* clock() { return time_controller_.GetClock(); }
1587 void AdvanceTime(TimeDelta duration) {
1588 time_controller_.AdvanceTime(duration);
1589 }
1590
1591 int64_t CurrentTimeMs() { return clock()->CurrentTime().ms(); }
1592
1593 protected:
1594 virtual TaskQueueFactory* GetTaskQueueFactory() {
1595 return time_controller_.GetTaskQueueFactory();
1596 }
1597
Jonas Orelandc7f691a2022-03-09 15:12:07 +01001598 test::ScopedKeyValueConfig field_trials_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001599 GlobalSimulatedTimeController time_controller_{Timestamp::Micros(1234)};
perkj26091b12016-09-01 01:17:40 -07001600 VideoSendStream::Config video_send_config_;
Erik Språng08127a92016-11-16 16:41:30 +01001601 VideoEncoderConfig video_encoder_config_;
Per512ecb32016-09-23 15:52:06 +02001602 int codec_width_;
1603 int codec_height_;
sprang4847ae62017-06-27 07:06:52 -07001604 int max_framerate_;
perkj26091b12016-09-01 01:17:40 -07001605 TestEncoder fake_encoder_;
Niels Möllercbcbc222018-09-28 09:07:24 +02001606 test::VideoEncoderProxyFactory encoder_factory_;
Sergey Silkin5ee69672019-07-02 14:18:34 +02001607 VideoBitrateAllocatorProxyFactory bitrate_allocator_factory_;
sprangc5d62e22017-04-02 23:53:04 -07001608 std::unique_ptr<MockableSendStatisticsProxy> stats_proxy_;
perkj26091b12016-09-01 01:17:40 -07001609 TestSink sink_;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001610 AdaptingFrameForwarder video_source_{&time_controller_};
mflodmancc3d4422017-08-03 08:27:51 -07001611 std::unique_ptr<VideoStreamEncoderUnderTest> video_stream_encoder_;
perkj26091b12016-09-01 01:17:40 -07001612};
1613
mflodmancc3d4422017-08-03 08:27:51 -07001614TEST_F(VideoStreamEncoderTest, EncodeOneFrame) {
Henrik Boström381d1092020-05-12 18:49:07 +02001615 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001616 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerc572ff32018-11-07 08:43:50 +01001617 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001618 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
sprang4847ae62017-06-27 07:06:52 -07001619 WaitForEncodedFrame(1);
perkja49cbd32016-09-16 07:53:41 -07001620 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
mflodmancc3d4422017-08-03 08:27:51 -07001621 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001622}
1623
mflodmancc3d4422017-08-03 08:27:51 -07001624TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
perkj26091b12016-09-01 01:17:40 -07001625 // Dropped since no target bitrate has been set.
Niels Möllerc572ff32018-11-07 08:43:50 +01001626 rtc::Event frame_destroyed_event;
Sebastian Janssona3177052018-04-10 13:05:49 +02001627 // The encoder will cache up to one frame for a short duration. Adding two
1628 // frames means that the first frame will be dropped and the second frame will
1629 // be sent when the encoder is enabled.
perkja49cbd32016-09-16 07:53:41 -07001630 video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event));
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001631 AdvanceTime(TimeDelta::Millis(10));
Sebastian Janssona3177052018-04-10 13:05:49 +02001632 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Markus Handell28c71802021-11-08 10:11:55 +01001633 AdvanceTime(TimeDelta::Zero());
perkja49cbd32016-09-16 07:53:41 -07001634 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001635
Henrik Boström381d1092020-05-12 18:49:07 +02001636 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001637 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001638
Sebastian Janssona3177052018-04-10 13:05:49 +02001639 // The pending frame should be received.
sprang4847ae62017-06-27 07:06:52 -07001640 WaitForEncodedFrame(2);
Sebastian Janssona3177052018-04-10 13:05:49 +02001641 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1642
1643 WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07001644 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001645}
1646
mflodmancc3d4422017-08-03 08:27:51 -07001647TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
Henrik Boström381d1092020-05-12 18:49:07 +02001648 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001649 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001650 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001651 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001652
Henrik Boström381d1092020-05-12 18:49:07 +02001653 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001654 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
1655
Sebastian Janssona3177052018-04-10 13:05:49 +02001656 // The encoder will cache up to one frame for a short duration. Adding two
1657 // frames means that the first frame will be dropped and the second frame will
1658 // be sent when the encoder is resumed.
perkja49cbd32016-09-16 07:53:41 -07001659 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sebastian Janssona3177052018-04-10 13:05:49 +02001660 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
perkj26091b12016-09-01 01:17:40 -07001661
Henrik Boström381d1092020-05-12 18:49:07 +02001662 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001663 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07001664 WaitForEncodedFrame(3);
Sebastian Janssona3177052018-04-10 13:05:49 +02001665 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1666 WaitForEncodedFrame(4);
mflodmancc3d4422017-08-03 08:27:51 -07001667 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001668}
1669
mflodmancc3d4422017-08-03 08:27:51 -07001670TEST_F(VideoStreamEncoderTest, DropsFramesWithSameOrOldNtpTimestamp) {
Henrik Boström381d1092020-05-12 18:49:07 +02001671 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001672 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkja49cbd32016-09-16 07:53:41 -07001673 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001674 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001675
1676 // This frame will be dropped since it has the same ntp timestamp.
perkja49cbd32016-09-16 07:53:41 -07001677 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
perkj26091b12016-09-01 01:17:40 -07001678
perkja49cbd32016-09-16 07:53:41 -07001679 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001680 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001681 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001682}
1683
mflodmancc3d4422017-08-03 08:27:51 -07001684TEST_F(VideoStreamEncoderTest, DropsFrameAfterStop) {
Henrik Boström381d1092020-05-12 18:49:07 +02001685 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001686 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001687
perkja49cbd32016-09-16 07:53:41 -07001688 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001689 WaitForEncodedFrame(1);
perkj26091b12016-09-01 01:17:40 -07001690
mflodmancc3d4422017-08-03 08:27:51 -07001691 video_stream_encoder_->Stop();
perkj26091b12016-09-01 01:17:40 -07001692 sink_.SetExpectNoFrames();
Niels Möllerc572ff32018-11-07 08:43:50 +01001693 rtc::Event frame_destroyed_event;
perkja49cbd32016-09-16 07:53:41 -07001694 video_source_.IncomingCapturedFrame(CreateFrame(2, &frame_destroyed_event));
1695 EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeoutMs));
perkj26091b12016-09-01 01:17:40 -07001696}
1697
Markus Handell9a478b52021-11-18 16:07:01 +01001698TEST_F(VideoStreamEncoderTest, DropsPendingFramesOnSlowEncode) {
1699 test::FrameForwarder source;
1700 video_stream_encoder_->SetSource(&source,
1701 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02001702 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001703 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj26091b12016-09-01 01:17:40 -07001704
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001705 int dropped_count = 0;
1706 stats_proxy_->SetDroppedFrameCallback(
1707 [&dropped_count](VideoStreamEncoderObserver::DropReason) {
1708 ++dropped_count;
1709 });
1710
Markus Handell9a478b52021-11-18 16:07:01 +01001711 source.IncomingCapturedFrame(CreateFrame(1, nullptr));
1712 source.IncomingCapturedFrame(CreateFrame(2, nullptr));
1713 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07001714 video_stream_encoder_->Stop();
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02001715 EXPECT_EQ(1, dropped_count);
perkj26091b12016-09-01 01:17:40 -07001716}
1717
Henrik Boström56db9ff2021-03-24 09:06:45 +01001718TEST_F(VideoStreamEncoderTest, NativeFrameWithoutI420SupportGetsDelivered) {
Henrik Boström381d1092020-05-12 18:49:07 +02001719 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001720 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001721
1722 rtc::Event frame_destroyed_event;
1723 video_source_.IncomingCapturedFrame(
1724 CreateFakeNativeFrame(1, &frame_destroyed_event));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001725 WaitForEncodedFrame(1);
1726 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1727 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001728 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1729 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001730 video_stream_encoder_->Stop();
1731}
1732
Henrik Boström56db9ff2021-03-24 09:06:45 +01001733TEST_F(VideoStreamEncoderTest,
1734 NativeFrameWithoutI420SupportGetsCroppedIfNecessary) {
Noah Richards51db4212019-06-12 06:59:12 -07001735 // Use the cropping factory.
1736 video_encoder_config_.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02001737 rtc::make_ref_counted<CroppingVideoStreamFactory>();
Noah Richards51db4212019-06-12 06:59:12 -07001738 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config_),
1739 kMaxPayloadLength);
1740 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
1741
1742 // Capture a frame at codec_width_/codec_height_.
Henrik Boström381d1092020-05-12 18:49:07 +02001743 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001744 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Noah Richards51db4212019-06-12 06:59:12 -07001745 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1746 WaitForEncodedFrame(1);
1747 // The encoder will have been configured once.
1748 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001749 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1750 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Noah Richards51db4212019-06-12 06:59:12 -07001751
1752 // Now send in a fake frame that needs to be cropped as the width/height
1753 // aren't divisible by 4 (see CreateEncoderStreams above).
1754 rtc::Event frame_destroyed_event;
1755 video_source_.IncomingCapturedFrame(CreateFakeNativeFrame(
1756 2, &frame_destroyed_event, codec_width_ + 1, codec_height_ + 1));
Henrik Boström56db9ff2021-03-24 09:06:45 +01001757 WaitForEncodedFrame(2);
1758 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
1759 fake_encoder_.GetLastInputPixelFormat());
Asa Persson606d3cb2021-10-04 10:07:11 +02001760 EXPECT_EQ(fake_encoder_.config().width, fake_encoder_.GetLastInputWidth());
1761 EXPECT_EQ(fake_encoder_.config().height, fake_encoder_.GetLastInputHeight());
Noah Richards51db4212019-06-12 06:59:12 -07001762 video_stream_encoder_->Stop();
1763}
1764
Evan Shrubsole895556e2020-10-05 09:15:13 +02001765TEST_F(VideoStreamEncoderTest, NonI420FramesShouldNotBeConvertedToI420) {
1766 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001767 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001768
1769 video_source_.IncomingCapturedFrame(
1770 CreateNV12Frame(1, codec_width_, codec_height_));
1771 WaitForEncodedFrame(1);
1772 EXPECT_EQ(VideoFrameBuffer::Type::kNV12,
1773 fake_encoder_.GetLastInputPixelFormat());
1774 video_stream_encoder_->Stop();
1775}
1776
Henrik Boström56db9ff2021-03-24 09:06:45 +01001777TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_NoFrameTypePreference) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001778 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001779 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001780
1781 fake_encoder_.SetPreferredPixelFormats({});
1782
1783 rtc::Event frame_destroyed_event;
1784 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1785 1, &frame_destroyed_event, codec_width_, codec_height_));
1786 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001787 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001788 fake_encoder_.GetLastInputPixelFormat());
1789 video_stream_encoder_->Stop();
1790}
1791
Henrik Boström56db9ff2021-03-24 09:06:45 +01001792TEST_F(VideoStreamEncoderTest,
1793 NativeFrameGetsDelivered_PixelFormatPreferenceMatches) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001794 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001795 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001796
1797 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kNV12});
1798
1799 rtc::Event frame_destroyed_event;
1800 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1801 1, &frame_destroyed_event, codec_width_, codec_height_));
1802 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001803 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001804 fake_encoder_.GetLastInputPixelFormat());
1805 video_stream_encoder_->Stop();
1806}
1807
Henrik Boström56db9ff2021-03-24 09:06:45 +01001808TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_MappingIsNotFeasible) {
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001809 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001810 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001811
1812 // Fake NV12 native frame does not allow mapping to I444.
1813 fake_encoder_.SetPreferredPixelFormats({VideoFrameBuffer::Type::kI444});
1814
1815 rtc::Event frame_destroyed_event;
1816 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1817 1, &frame_destroyed_event, codec_width_, codec_height_));
1818 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001819 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsoleb556b082020-10-08 14:56:45 +02001820 fake_encoder_.GetLastInputPixelFormat());
1821 video_stream_encoder_->Stop();
1822}
1823
Henrik Boström56db9ff2021-03-24 09:06:45 +01001824TEST_F(VideoStreamEncoderTest, NativeFrameGetsDelivered_BackedByNV12) {
Evan Shrubsole895556e2020-10-05 09:15:13 +02001825 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001826 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole895556e2020-10-05 09:15:13 +02001827
1828 rtc::Event frame_destroyed_event;
1829 video_source_.IncomingCapturedFrame(CreateFakeNV12NativeFrame(
1830 1, &frame_destroyed_event, codec_width_, codec_height_));
1831 WaitForEncodedFrame(1);
Henrik Boström56db9ff2021-03-24 09:06:45 +01001832 EXPECT_EQ(VideoFrameBuffer::Type::kNative,
Evan Shrubsole895556e2020-10-05 09:15:13 +02001833 fake_encoder_.GetLastInputPixelFormat());
1834 video_stream_encoder_->Stop();
1835}
1836
Ying Wang9b881ab2020-02-07 14:29:32 +01001837TEST_F(VideoStreamEncoderTest, DropsFramesWhenCongestionWindowPushbackSet) {
Henrik Boström381d1092020-05-12 18:49:07 +02001838 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001839 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ying Wang9b881ab2020-02-07 14:29:32 +01001840 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1841 WaitForEncodedFrame(1);
1842
Henrik Boström381d1092020-05-12 18:49:07 +02001843 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001844 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0.5);
Ying Wang9b881ab2020-02-07 14:29:32 +01001845 // The congestion window pushback is set to 0.5, which will drop 1/2 of
1846 // frames. Adding two frames means that the first frame will be dropped and
1847 // the second frame will be sent to the encoder.
1848 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1849 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
1850 WaitForEncodedFrame(3);
1851 video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr));
1852 video_source_.IncomingCapturedFrame(CreateFrame(5, nullptr));
1853 WaitForEncodedFrame(5);
1854 EXPECT_EQ(2u, stats_proxy_->GetStats().frames_dropped_by_congestion_window);
1855 video_stream_encoder_->Stop();
1856}
1857
mflodmancc3d4422017-08-03 08:27:51 -07001858TEST_F(VideoStreamEncoderTest,
1859 ConfigureEncoderTriggersOnEncoderConfigurationChanged) {
Henrik Boström381d1092020-05-12 18:49:07 +02001860 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001861 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per21d45d22016-10-30 21:37:57 +01001862 EXPECT_EQ(0, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001863
1864 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001865 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001866 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001867 // The encoder will have been configured once when the first frame is
1868 // received.
1869 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Per512ecb32016-09-23 15:52:06 +02001870
1871 VideoEncoderConfig video_encoder_config;
Niels Möller259a4972018-04-05 15:36:51 +02001872 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Per512ecb32016-09-23 15:52:06 +02001873 video_encoder_config.min_transmit_bitrate_bps = 9999;
mflodmancc3d4422017-08-03 08:27:51 -07001874 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02001875 kMaxPayloadLength);
Per512ecb32016-09-23 15:52:06 +02001876
1877 // Capture a frame and wait for it to synchronize with the encoder thread.
Perf8c5f2b2016-09-23 16:24:55 +02001878 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001879 WaitForEncodedFrame(2);
Per21d45d22016-10-30 21:37:57 +01001880 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkj3b703ed2016-09-29 23:25:40 -07001881 EXPECT_EQ(9999, sink_.last_min_transmit_bitrate());
perkj26105b42016-09-29 22:39:10 -07001882
mflodmancc3d4422017-08-03 08:27:51 -07001883 video_stream_encoder_->Stop();
perkj26105b42016-09-29 22:39:10 -07001884}
1885
mflodmancc3d4422017-08-03 08:27:51 -07001886TEST_F(VideoStreamEncoderTest, FrameResolutionChangeReconfigureEncoder) {
Henrik Boström381d1092020-05-12 18:49:07 +02001887 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001888 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkjfa10b552016-10-02 23:45:26 -07001889
1890 // Capture a frame and wait for it to synchronize with the encoder thread.
1891 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001892 WaitForEncodedFrame(1);
Per21d45d22016-10-30 21:37:57 +01001893 // The encoder will have been configured once.
1894 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001895 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1896 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
perkjfa10b552016-10-02 23:45:26 -07001897
1898 codec_width_ *= 2;
1899 codec_height_ *= 2;
1900 // Capture a frame with a higher resolution and wait for it to synchronize
1901 // with the encoder thread.
1902 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
sprang4847ae62017-06-27 07:06:52 -07001903 WaitForEncodedFrame(2);
Asa Persson606d3cb2021-10-04 10:07:11 +02001904 EXPECT_EQ(codec_width_, fake_encoder_.config().width);
1905 EXPECT_EQ(codec_height_, fake_encoder_.config().height);
Per21d45d22016-10-30 21:37:57 +01001906 EXPECT_EQ(2, sink_.number_of_reconfigurations());
perkjfa10b552016-10-02 23:45:26 -07001907
mflodmancc3d4422017-08-03 08:27:51 -07001908 video_stream_encoder_->Stop();
perkjfa10b552016-10-02 23:45:26 -07001909}
1910
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001911TEST_F(VideoStreamEncoderTest,
1912 EncoderInstanceDestroyedBeforeAnotherInstanceCreated) {
Henrik Boström381d1092020-05-12 18:49:07 +02001913 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001914 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin443b7ee2019-06-28 12:53:07 +02001915
1916 // Capture a frame and wait for it to synchronize with the encoder thread.
1917 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1918 WaitForEncodedFrame(1);
1919
1920 VideoEncoderConfig video_encoder_config;
1921 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1922 // Changing the max payload data length recreates encoder.
1923 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1924 kMaxPayloadLength / 2);
1925
1926 // Capture a frame and wait for it to synchronize with the encoder thread.
1927 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1928 WaitForEncodedFrame(2);
1929 EXPECT_EQ(1, encoder_factory_.GetMaxNumberOfSimultaneousEncoderInstances());
1930
1931 video_stream_encoder_->Stop();
1932}
1933
Sergey Silkin5ee69672019-07-02 14:18:34 +02001934TEST_F(VideoStreamEncoderTest, BitrateLimitsChangeReconfigureRateAllocator) {
Henrik Boström381d1092020-05-12 18:49:07 +02001935 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001936 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001937
1938 VideoEncoderConfig video_encoder_config;
1939 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02001940 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
1941 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001942 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1943 kMaxPayloadLength);
1944
1945 // Capture a frame and wait for it to synchronize with the encoder thread.
1946 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
1947 WaitForEncodedFrame(1);
1948 // The encoder will have been configured once when the first frame is
1949 // received.
1950 EXPECT_EQ(1, sink_.number_of_reconfigurations());
Asa Persson606d3cb2021-10-04 10:07:11 +02001951 EXPECT_EQ(kTargetBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001952 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001953 EXPECT_EQ(kStartBitrate.bps(),
Sergey Silkin5ee69672019-07-02 14:18:34 +02001954 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
1955
Sergey Silkin6456e352019-07-08 17:56:40 +02001956 test::FillEncoderConfiguration(kVideoCodecVP8, 1,
1957 &video_encoder_config); //???
Asa Persson606d3cb2021-10-04 10:07:11 +02001958 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps() * 2;
1959 video_stream_encoder_->SetStartBitrate(kStartBitrate.bps() * 2);
Sergey Silkin5ee69672019-07-02 14:18:34 +02001960 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
1961 kMaxPayloadLength);
1962
1963 // Capture a frame and wait for it to synchronize with the encoder thread.
1964 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
1965 WaitForEncodedFrame(2);
1966 EXPECT_EQ(2, sink_.number_of_reconfigurations());
1967 // Bitrate limits have changed - rate allocator should be reconfigured,
1968 // encoder should not be reconfigured.
Asa Persson606d3cb2021-10-04 10:07:11 +02001969 EXPECT_EQ(kTargetBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001970 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001971 EXPECT_EQ(kStartBitrate.bps() * 2,
Sergey Silkin5ee69672019-07-02 14:18:34 +02001972 bitrate_allocator_factory_.codec_config().startBitrate * 1000);
Asa Persson606d3cb2021-10-04 10:07:11 +02001973 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Sergey Silkin5ee69672019-07-02 14:18:34 +02001974
1975 video_stream_encoder_->Stop();
1976}
1977
Sergey Silkin6456e352019-07-08 17:56:40 +02001978TEST_F(VideoStreamEncoderTest,
Sergey Silkincd02eba2020-01-20 14:48:40 +01001979 IntersectionOfEncoderAndAppBitrateLimitsUsedWhenBothProvided) {
Henrik Boström381d1092020-05-12 18:49:07 +02001980 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02001981 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02001982
Sergey Silkincd02eba2020-01-20 14:48:40 +01001983 const uint32_t kMinEncBitrateKbps = 100;
1984 const uint32_t kMaxEncBitrateKbps = 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001985 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
Sergey Silkincd02eba2020-01-20 14:48:40 +01001986 /*frame_size_pixels=*/codec_width_ * codec_height_,
1987 /*min_start_bitrate_bps=*/0,
1988 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
1989 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02001990 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
1991
Sergey Silkincd02eba2020-01-20 14:48:40 +01001992 VideoEncoderConfig video_encoder_config;
1993 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
1994 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps + 1) * 1000;
1995 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
1996 (kMinEncBitrateKbps + 1) * 1000;
1997 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
1998 kMaxPayloadLength);
1999
2000 // When both encoder and app provide bitrate limits, the intersection of
2001 // provided sets should be used.
2002 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2003 WaitForEncodedFrame(1);
2004 EXPECT_EQ(kMaxEncBitrateKbps,
2005 bitrate_allocator_factory_.codec_config().maxBitrate);
2006 EXPECT_EQ(kMinEncBitrateKbps + 1,
2007 bitrate_allocator_factory_.codec_config().minBitrate);
2008
2009 video_encoder_config.max_bitrate_bps = (kMaxEncBitrateKbps - 1) * 1000;
2010 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2011 (kMinEncBitrateKbps - 1) * 1000;
2012 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2013 kMaxPayloadLength);
2014 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002015 WaitForEncodedFrame(2);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002016 EXPECT_EQ(kMaxEncBitrateKbps - 1,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002017 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002018 EXPECT_EQ(kMinEncBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002019 bitrate_allocator_factory_.codec_config().minBitrate);
2020
Sergey Silkincd02eba2020-01-20 14:48:40 +01002021 video_stream_encoder_->Stop();
2022}
2023
2024TEST_F(VideoStreamEncoderTest,
2025 EncoderAndAppLimitsDontIntersectEncoderLimitsIgnored) {
Henrik Boström381d1092020-05-12 18:49:07 +02002026 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002027 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002028
2029 const uint32_t kMinAppBitrateKbps = 100;
2030 const uint32_t kMaxAppBitrateKbps = 200;
2031 const uint32_t kMinEncBitrateKbps = kMaxAppBitrateKbps + 1;
2032 const uint32_t kMaxEncBitrateKbps = kMaxAppBitrateKbps * 2;
2033 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2034 /*frame_size_pixels=*/codec_width_ * codec_height_,
2035 /*min_start_bitrate_bps=*/0,
2036 /*min_bitrate_bps=*/kMinEncBitrateKbps * 1000,
2037 /*max_bitrate_bps=*/kMaxEncBitrateKbps * 1000);
2038 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2039
2040 VideoEncoderConfig video_encoder_config;
2041 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2042 video_encoder_config.max_bitrate_bps = kMaxAppBitrateKbps * 1000;
2043 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
2044 kMinAppBitrateKbps * 1000;
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002045 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2046 kMaxPayloadLength);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002047
Sergey Silkincd02eba2020-01-20 14:48:40 +01002048 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
2049 WaitForEncodedFrame(1);
2050 EXPECT_EQ(kMaxAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002051 bitrate_allocator_factory_.codec_config().maxBitrate);
Sergey Silkincd02eba2020-01-20 14:48:40 +01002052 EXPECT_EQ(kMinAppBitrateKbps,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002053 bitrate_allocator_factory_.codec_config().minBitrate);
Sergey Silkin6456e352019-07-08 17:56:40 +02002054
2055 video_stream_encoder_->Stop();
2056}
2057
2058TEST_F(VideoStreamEncoderTest,
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002059 EncoderRecommendedMaxAndMinBitratesUsedForGivenResolution) {
Henrik Boström381d1092020-05-12 18:49:07 +02002060 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002061 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6456e352019-07-08 17:56:40 +02002062
2063 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_270p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002064 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002065 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits_360p(
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002066 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002067 fake_encoder_.SetResolutionBitrateLimits(
2068 {encoder_bitrate_limits_270p, encoder_bitrate_limits_360p});
2069
2070 VideoEncoderConfig video_encoder_config;
2071 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2072 video_encoder_config.max_bitrate_bps = 0;
2073 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2074 kMaxPayloadLength);
2075
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002076 // 270p. The bitrate limits recommended by encoder for 270p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002077 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2078 WaitForEncodedFrame(1);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002079 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2080 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002081 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2082 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2083
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002084 // 360p. The bitrate limits recommended by encoder for 360p should be used.
Sergey Silkin6456e352019-07-08 17:56:40 +02002085 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2086 WaitForEncodedFrame(2);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002087 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2088 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002089 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2090 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2091
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002092 // Resolution between 270p and 360p. The bitrate limits recommended by
Sergey Silkin6456e352019-07-08 17:56:40 +02002093 // encoder for 360p should be used.
2094 video_source_.IncomingCapturedFrame(
2095 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2096 WaitForEncodedFrame(3);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002097 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2098 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002099 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2100 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2101
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002102 // Resolution higher than 360p. The caps recommended by encoder should be
Sergey Silkin6456e352019-07-08 17:56:40 +02002103 // ignored.
2104 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2105 WaitForEncodedFrame(4);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002106 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2107 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002108 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2109 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002110 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.min_bitrate_bps),
2111 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002112 EXPECT_NE(static_cast<uint32_t>(encoder_bitrate_limits_360p.max_bitrate_bps),
2113 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2114
2115 // Resolution lower than 270p. The max bitrate limit recommended by encoder
2116 // for 270p should be used.
2117 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2118 WaitForEncodedFrame(5);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002119 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.min_bitrate_bps),
2120 bitrate_allocator_factory_.codec_config().minBitrate * 1000);
Sergey Silkin6456e352019-07-08 17:56:40 +02002121 EXPECT_EQ(static_cast<uint32_t>(encoder_bitrate_limits_270p.max_bitrate_bps),
2122 bitrate_allocator_factory_.codec_config().maxBitrate * 1000);
2123
2124 video_stream_encoder_->Stop();
2125}
2126
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002127TEST_F(VideoStreamEncoderTest, EncoderRecommendedMaxBitrateCapsTargetBitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02002128 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002129 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkin6b2cec12019-08-09 16:04:05 +02002130
2131 VideoEncoderConfig video_encoder_config;
2132 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
2133 video_encoder_config.max_bitrate_bps = 0;
2134 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
2135 kMaxPayloadLength);
2136
2137 // Encode 720p frame to get the default encoder target bitrate.
2138 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2139 WaitForEncodedFrame(1);
2140 const uint32_t kDefaultTargetBitrateFor720pKbps =
2141 bitrate_allocator_factory_.codec_config()
2142 .simulcastStream[0]
2143 .targetBitrate;
2144
2145 // Set the max recommended encoder bitrate to something lower than the default
2146 // target bitrate.
2147 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
2148 1280 * 720, 10 * 1000, 10 * 1000,
2149 kDefaultTargetBitrateFor720pKbps / 2 * 1000);
2150 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
2151
2152 // Change resolution to trigger encoder reinitialization.
2153 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2154 WaitForEncodedFrame(2);
2155 video_source_.IncomingCapturedFrame(CreateFrame(3, 1280, 720));
2156 WaitForEncodedFrame(3);
2157
2158 // Ensure the target bitrate is capped by the max bitrate.
2159 EXPECT_EQ(bitrate_allocator_factory_.codec_config().maxBitrate * 1000,
2160 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2161 EXPECT_EQ(bitrate_allocator_factory_.codec_config()
2162 .simulcastStream[0]
2163 .targetBitrate *
2164 1000,
2165 static_cast<uint32_t>(encoder_bitrate_limits.max_bitrate_bps));
2166
2167 video_stream_encoder_->Stop();
2168}
2169
Åsa Perssona7e34d32021-01-20 15:36:13 +01002170TEST_F(VideoStreamEncoderTest,
2171 EncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2172 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2173 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2174 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2175 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2176 fake_encoder_.SetResolutionBitrateLimits(
2177 {kEncoderLimits270p, kEncoderLimits360p});
2178
2179 // Two streams, highest stream active.
2180 VideoEncoderConfig config;
2181 const int kNumStreams = 2;
2182 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2183 config.max_bitrate_bps = 0;
2184 config.simulcast_layers[0].active = false;
2185 config.simulcast_layers[1].active = true;
2186 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002187 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002188 "VP8", /*max qp*/ 56, /*screencast*/ false,
2189 /*screenshare enabled*/ false);
2190 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2191
2192 // The encoder bitrate limits for 270p should be used.
2193 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2194 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002195 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002196 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002197 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002198 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002199 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002200
2201 // The encoder bitrate limits for 360p should be used.
2202 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2203 EXPECT_FALSE(WaitForFrame(1000));
2204 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002205 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002206 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002207 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002208
2209 // Resolution b/w 270p and 360p. The encoder limits for 360p should be used.
2210 video_source_.IncomingCapturedFrame(
2211 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2212 EXPECT_FALSE(WaitForFrame(1000));
2213 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002214 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002215 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002216 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002217
2218 // Resolution higher than 360p. Encoder limits should be ignored.
2219 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2220 EXPECT_FALSE(WaitForFrame(1000));
2221 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002222 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002223 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002224 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002225 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002226 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002227 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002228 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002229
2230 // Resolution lower than 270p. The encoder limits for 270p should be used.
2231 video_source_.IncomingCapturedFrame(CreateFrame(5, 320, 180));
2232 EXPECT_FALSE(WaitForFrame(1000));
2233 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002234 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002235 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002236 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002237
2238 video_stream_encoder_->Stop();
2239}
2240
2241TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01002242 DefaultEncoderMaxAndMinBitratesUsedForTwoStreamsHighestActive) {
2243 // Two streams, highest stream active.
2244 VideoEncoderConfig config;
2245 const int kNumStreams = 2;
2246 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2247 config.max_bitrate_bps = 0;
2248 config.simulcast_layers[0].active = false;
2249 config.simulcast_layers[1].active = true;
2250 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002251 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson258e9892021-02-25 10:39:51 +01002252 "VP8", /*max qp*/ 56, /*screencast*/ false,
2253 /*screenshare enabled*/ false);
2254 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2255
2256 // Default bitrate limits for 270p should be used.
2257 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2258 kDefaultLimits270p =
2259 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002260 kVideoCodecVP8, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01002261 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2262 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002263 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Persson258e9892021-02-25 10:39:51 +01002264 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002265 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002266 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002267 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002268
2269 // Default bitrate limits for 360p should be used.
2270 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2271 kDefaultLimits360p =
2272 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002273 kVideoCodecVP8, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01002274 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2275 EXPECT_FALSE(WaitForFrame(1000));
2276 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002277 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002278 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002279 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002280
2281 // Resolution b/w 270p and 360p. The default limits for 360p should be used.
2282 video_source_.IncomingCapturedFrame(
2283 CreateFrame(3, (640 + 480) / 2, (360 + 270) / 2));
2284 EXPECT_FALSE(WaitForFrame(1000));
2285 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002286 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002287 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002288 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002289
2290 // Default bitrate limits for 540p should be used.
2291 const absl::optional<VideoEncoder::ResolutionBitrateLimits>
2292 kDefaultLimits540p =
2293 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
Sergey Silkina86b29b2021-03-05 13:29:19 +01002294 kVideoCodecVP8, 960 * 540);
Åsa Persson258e9892021-02-25 10:39:51 +01002295 video_source_.IncomingCapturedFrame(CreateFrame(4, 960, 540));
2296 EXPECT_FALSE(WaitForFrame(1000));
2297 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002298 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002299 EXPECT_EQ(static_cast<uint32_t>(kDefaultLimits540p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002300 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01002301
2302 video_stream_encoder_->Stop();
2303}
2304
2305TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01002306 EncoderMaxAndMinBitratesUsedForThreeStreamsMiddleActive) {
2307 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2308 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2309 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2310 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2311 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2312 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2313 fake_encoder_.SetResolutionBitrateLimits(
2314 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2315
2316 // Three streams, middle stream active.
2317 VideoEncoderConfig config;
2318 const int kNumStreams = 3;
2319 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2320 config.simulcast_layers[0].active = false;
2321 config.simulcast_layers[1].active = true;
2322 config.simulcast_layers[2].active = false;
2323 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002324 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002325 "VP8", /*max qp*/ 56, /*screencast*/ false,
2326 /*screenshare enabled*/ false);
2327 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2328
2329 // The encoder bitrate limits for 360p should be used.
2330 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2331 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002332 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002333 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002334 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002335 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002336 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002337
2338 // The encoder bitrate limits for 270p should be used.
2339 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
2340 EXPECT_FALSE(WaitForFrame(1000));
2341 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002342 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002343 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002344 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002345
2346 video_stream_encoder_->Stop();
2347}
2348
2349TEST_F(VideoStreamEncoderTest,
2350 EncoderMaxAndMinBitratesNotUsedForThreeStreamsLowestActive) {
2351 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2352 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2353 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2354 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2355 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
2356 1280 * 720, 54 * 1000, 31 * 1000, 3456 * 1000);
2357 fake_encoder_.SetResolutionBitrateLimits(
2358 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
2359
2360 // Three streams, lowest stream active.
2361 VideoEncoderConfig config;
2362 const int kNumStreams = 3;
2363 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2364 config.simulcast_layers[0].active = true;
2365 config.simulcast_layers[1].active = false;
2366 config.simulcast_layers[2].active = false;
2367 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002368 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002369 "VP8", /*max qp*/ 56, /*screencast*/ false,
2370 /*screenshare enabled*/ false);
2371 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2372
2373 // Resolution on lowest stream lower than 270p. The encoder limits not applied
2374 // on lowest stream, limits for 270p should not be used
2375 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
2376 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002377 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002378 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002379 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002380 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002381 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002382
2383 video_stream_encoder_->Stop();
2384}
2385
2386TEST_F(VideoStreamEncoderTest,
2387 EncoderMaxBitrateCappedByConfigForTwoStreamsHighestActive) {
2388 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
2389 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
2390 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
2391 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
2392 fake_encoder_.SetResolutionBitrateLimits(
2393 {kEncoderLimits270p, kEncoderLimits360p});
2394 const int kMaxBitrateBps = kEncoderLimits360p.max_bitrate_bps - 100 * 1000;
2395
2396 // Two streams, highest stream active.
2397 VideoEncoderConfig config;
2398 const int kNumStreams = 2;
2399 test::FillEncoderConfiguration(kVideoCodecVP8, kNumStreams, &config);
2400 config.simulcast_layers[0].active = false;
2401 config.simulcast_layers[1].active = true;
2402 config.simulcast_layers[1].max_bitrate_bps = kMaxBitrateBps;
2403 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002404 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssona7e34d32021-01-20 15:36:13 +01002405 "VP8", /*max qp*/ 56, /*screencast*/ false,
2406 /*screenshare enabled*/ false);
2407 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
2408
2409 // The encoder bitrate limits for 270p should be used.
2410 video_source_.IncomingCapturedFrame(CreateFrame(1, 480, 270));
2411 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02002412 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, kNumStreams);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002413 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002414 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002415 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002416 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002417
2418 // The max configured bitrate is less than the encoder limit for 360p.
2419 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
2420 EXPECT_FALSE(WaitForFrame(1000));
2421 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002422 fake_encoder_.config().simulcastStream[1].minBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002423 EXPECT_EQ(static_cast<uint32_t>(kMaxBitrateBps),
Asa Persson606d3cb2021-10-04 10:07:11 +02002424 fake_encoder_.config().simulcastStream[1].maxBitrate * 1000);
Åsa Perssona7e34d32021-01-20 15:36:13 +01002425
2426 video_stream_encoder_->Stop();
2427}
2428
mflodmancc3d4422017-08-03 08:27:51 -07002429TEST_F(VideoStreamEncoderTest, SwitchSourceDeregisterEncoderAsSink) {
perkj803d97f2016-11-01 11:45:46 -07002430 EXPECT_TRUE(video_source_.has_sinks());
2431 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002432 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002433 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07002434 EXPECT_FALSE(video_source_.has_sinks());
2435 EXPECT_TRUE(new_video_source.has_sinks());
2436
mflodmancc3d4422017-08-03 08:27:51 -07002437 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002438}
2439
mflodmancc3d4422017-08-03 08:27:51 -07002440TEST_F(VideoStreamEncoderTest, SinkWantsRotationApplied) {
perkj803d97f2016-11-01 11:45:46 -07002441 EXPECT_FALSE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002442 video_stream_encoder_->SetSink(&sink_, true /*rotation_applied*/);
perkj803d97f2016-11-01 11:45:46 -07002443 EXPECT_TRUE(video_source_.sink_wants().rotation_applied);
mflodmancc3d4422017-08-03 08:27:51 -07002444 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002445}
2446
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002447class ResolutionAlignmentTest
2448 : public VideoStreamEncoderTest,
2449 public ::testing::WithParamInterface<
2450 ::testing::tuple<int, std::vector<double>>> {
2451 public:
2452 ResolutionAlignmentTest()
2453 : requested_alignment_(::testing::get<0>(GetParam())),
2454 scale_factors_(::testing::get<1>(GetParam())) {}
2455
2456 protected:
2457 const int requested_alignment_;
2458 const std::vector<double> scale_factors_;
2459};
2460
2461INSTANTIATE_TEST_SUITE_P(
2462 AlignmentAndScaleFactors,
2463 ResolutionAlignmentTest,
2464 ::testing::Combine(
2465 ::testing::Values(1, 2, 3, 4, 5, 6, 16, 22), // requested_alignment_
2466 ::testing::Values(std::vector<double>{-1.0}, // scale_factors_
2467 std::vector<double>{-1.0, -1.0},
2468 std::vector<double>{-1.0, -1.0, -1.0},
2469 std::vector<double>{4.0, 2.0, 1.0},
2470 std::vector<double>{9999.0, -1.0, 1.0},
2471 std::vector<double>{3.99, 2.01, 1.0},
2472 std::vector<double>{4.9, 1.7, 1.25},
2473 std::vector<double>{10.0, 4.0, 3.0},
2474 std::vector<double>{1.75, 3.5},
2475 std::vector<double>{1.5, 2.5},
2476 std::vector<double>{1.3, 1.0})));
2477
2478TEST_P(ResolutionAlignmentTest, SinkWantsAlignmentApplied) {
2479 // Set requested resolution alignment.
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002480 video_source_.set_adaptation_enabled(true);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002481 fake_encoder_.SetRequestedResolutionAlignment(requested_alignment_);
2482 fake_encoder_.SetApplyAlignmentToAllSimulcastLayers(true);
2483
2484 // Fill config with the scaling factor by which to reduce encoding size.
2485 const int num_streams = scale_factors_.size();
2486 VideoEncoderConfig config;
2487 test::FillEncoderConfiguration(kVideoCodecVP8, num_streams, &config);
2488 for (int i = 0; i < num_streams; ++i) {
2489 config.simulcast_layers[i].scale_resolution_down_by = scale_factors_[i];
2490 }
2491 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02002492 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002493 "VP8", /*max qp*/ 56, /*screencast*/ false,
2494 /*screenshare enabled*/ false);
2495 video_stream_encoder_->ConfigureEncoder(std::move(config), kMaxPayloadLength);
2496
Henrik Boström381d1092020-05-12 18:49:07 +02002497 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002498 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
2499 0, 0, 0);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002500 // Wait for all layers before triggering event.
2501 sink_.SetNumExpectedLayers(num_streams);
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002502
2503 // On the 1st frame, we should have initialized the encoder and
2504 // asked for its resolution requirements.
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002505 int64_t timestamp_ms = kFrameIntervalMs;
2506 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2507 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002508 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002509
2510 // On the 2nd frame, we should be receiving a correctly aligned resolution.
2511 // (It's up the to the encoder to potentially drop the previous frame,
2512 // to avoid coding back-to-back keyframes.)
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002513 timestamp_ms += kFrameIntervalMs;
2514 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
2515 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02002516 EXPECT_GE(fake_encoder_.GetNumInitializations(), 1);
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002517
Asa Persson606d3cb2021-10-04 10:07:11 +02002518 VideoCodec codec = fake_encoder_.config();
Åsa Perssonc5a74ff2020-09-20 17:50:00 +02002519 EXPECT_EQ(codec.numberOfSimulcastStreams, num_streams);
2520 // Frame size should be a multiple of the requested alignment.
2521 for (int i = 0; i < codec.numberOfSimulcastStreams; ++i) {
2522 EXPECT_EQ(codec.simulcastStream[i].width % requested_alignment_, 0);
2523 EXPECT_EQ(codec.simulcastStream[i].height % requested_alignment_, 0);
2524 // Aspect ratio should match.
2525 EXPECT_EQ(codec.width * codec.simulcastStream[i].height,
2526 codec.height * codec.simulcastStream[i].width);
2527 }
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002528
2529 video_stream_encoder_->Stop();
2530}
2531
Jonathan Yubc771b72017-12-08 17:04:29 -08002532TEST_F(VideoStreamEncoderTest, TestCpuDowngrades_BalancedMode) {
2533 const int kFramerateFps = 30;
asaperssonf7e294d2017-06-13 23:25:22 -07002534 const int kWidth = 1280;
2535 const int kHeight = 720;
Jonathan Yubc771b72017-12-08 17:04:29 -08002536
2537 // We rely on the automatic resolution adaptation, but we handle framerate
2538 // adaptation manually by mocking the stats proxy.
2539 video_source_.set_adaptation_enabled(true);
asaperssonf7e294d2017-06-13 23:25:22 -07002540
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002541 // Enable BALANCED preference, no initial limitation.
Henrik Boström381d1092020-05-12 18:49:07 +02002542 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002543 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002544 video_stream_encoder_->SetSource(&video_source_,
2545 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002546 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07002547 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002548 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07002549 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
2550
Jonathan Yubc771b72017-12-08 17:04:29 -08002551 // Adapt down as far as possible.
2552 rtc::VideoSinkWants last_wants;
2553 int64_t t = 1;
2554 int loop_count = 0;
2555 do {
2556 ++loop_count;
2557 last_wants = video_source_.sink_wants();
2558
2559 // Simulate the framerate we've been asked to adapt to.
2560 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2561 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2562 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2563 mock_stats.input_frame_rate = fps;
2564 stats_proxy_->SetMockStats(mock_stats);
2565
2566 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2567 sink_.WaitForEncodedFrame(t);
2568 t += frame_interval_ms;
2569
mflodmancc3d4422017-08-03 08:27:51 -07002570 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002571 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002572 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002573 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2574 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002575 } while (video_source_.sink_wants().max_pixel_count <
2576 last_wants.max_pixel_count ||
2577 video_source_.sink_wants().max_framerate_fps <
2578 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002579
Jonathan Yubc771b72017-12-08 17:04:29 -08002580 // Verify that we've adapted all the way down.
2581 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002582 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002583 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
2584 EXPECT_EQ(loop_count - 1,
asaperssonf7e294d2017-06-13 23:25:22 -07002585 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08002586 EXPECT_EQ(kMinPixelsPerFrame, *video_source_.last_sent_width() *
2587 *video_source_.last_sent_height());
2588 EXPECT_EQ(kMinBalancedFramerateFps,
2589 video_source_.sink_wants().max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002590
Jonathan Yubc771b72017-12-08 17:04:29 -08002591 // Adapt back up the same number of times we adapted down.
2592 for (int i = 0; i < loop_count - 1; ++i) {
2593 last_wants = video_source_.sink_wants();
2594
2595 // Simulate the framerate we've been asked to adapt to.
2596 const int fps = std::min(kFramerateFps, last_wants.max_framerate_fps);
2597 const int frame_interval_ms = rtc::kNumMillisecsPerSec / fps;
2598 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
2599 mock_stats.input_frame_rate = fps;
2600 stats_proxy_->SetMockStats(mock_stats);
2601
2602 video_source_.IncomingCapturedFrame(CreateFrame(t, kWidth, kHeight));
2603 sink_.WaitForEncodedFrame(t);
2604 t += frame_interval_ms;
2605
Henrik Boström91aa7322020-04-28 12:24:33 +02002606 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002607 EXPECT_THAT(
Jonathan Yubc771b72017-12-08 17:04:29 -08002608 video_source_.sink_wants(),
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002609 FpsInRangeForPixelsInBalanced(*video_source_.last_sent_width() *
2610 *video_source_.last_sent_height()));
Jonathan Yubc771b72017-12-08 17:04:29 -08002611 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count >
2612 last_wants.max_pixel_count ||
2613 video_source_.sink_wants().max_framerate_fps >
2614 last_wants.max_framerate_fps);
asaperssonf7e294d2017-06-13 23:25:22 -07002615 }
2616
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002617 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
Jonathan Yubc771b72017-12-08 17:04:29 -08002618 stats_proxy_->ResetMockStats();
asaperssonf7e294d2017-06-13 23:25:22 -07002619 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08002620 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
2621 EXPECT_EQ((loop_count - 1) * 2,
2622 stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07002623
mflodmancc3d4422017-08-03 08:27:51 -07002624 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07002625}
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01002626
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002627TEST_F(VideoStreamEncoderTest,
2628 SinkWantsNotChangedByResourceLimitedBeforeDegradationPreferenceChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02002629 video_stream_encoder_->OnBitrateUpdated(kTargetBitrate, kTargetBitrate,
2630 kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002631 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002632
2633 const int kFrameWidth = 1280;
2634 const int kFrameHeight = 720;
2635
2636 int64_t ntp_time = kFrameIntervalMs;
2637
2638 // Force an input frame rate to be available, or the adaptation call won't
2639 // know what framerate to adapt form.
2640 const int kInputFps = 30;
2641 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2642 stats.input_frame_rate = kInputFps;
2643 stats_proxy_->SetMockStats(stats);
2644
2645 video_source_.set_adaptation_enabled(true);
2646 video_stream_encoder_->SetSource(
2647 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002648 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002649 video_source_.IncomingCapturedFrame(
2650 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2651 sink_.WaitForEncodedFrame(ntp_time);
2652 ntp_time += kFrameIntervalMs;
2653
2654 // Trigger CPU overuse.
2655 video_stream_encoder_->TriggerCpuOveruse();
2656 video_source_.IncomingCapturedFrame(
2657 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2658 sink_.WaitForEncodedFrame(ntp_time);
2659 ntp_time += kFrameIntervalMs;
2660
2661 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2662 EXPECT_EQ(std::numeric_limits<int>::max(),
2663 video_source_.sink_wants().max_pixel_count);
2664 // Some framerate constraint should be set.
2665 int restricted_fps = video_source_.sink_wants().max_framerate_fps;
2666 EXPECT_LT(restricted_fps, kInputFps);
2667 video_source_.IncomingCapturedFrame(
2668 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2669 sink_.WaitForEncodedFrame(ntp_time);
2670 ntp_time += 100;
2671
Henrik Boström2671dac2020-05-19 16:29:09 +02002672 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002673 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
2674 // Give the encoder queue time to process the change in degradation preference
2675 // by waiting for an encoded frame.
2676 video_source_.IncomingCapturedFrame(
2677 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2678 sink_.WaitForEncodedFrame(ntp_time);
2679 ntp_time += kFrameIntervalMs;
2680
2681 video_stream_encoder_->TriggerQualityLow();
2682 video_source_.IncomingCapturedFrame(
2683 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2684 sink_.WaitForEncodedFrame(ntp_time);
2685 ntp_time += kFrameIntervalMs;
2686
2687 // Some resolution constraint should be set.
2688 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2689 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2690 kFrameWidth * kFrameHeight);
2691 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2692
2693 int pixel_count = video_source_.sink_wants().max_pixel_count;
2694 // Triggering a CPU underuse should not change the sink wants since it has
2695 // not been overused for resolution since we changed degradation preference.
2696 video_stream_encoder_->TriggerCpuUnderuse();
2697 video_source_.IncomingCapturedFrame(
2698 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2699 sink_.WaitForEncodedFrame(ntp_time);
2700 ntp_time += kFrameIntervalMs;
2701 EXPECT_EQ(video_source_.sink_wants().max_pixel_count, pixel_count);
2702 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, kInputFps);
2703
Evan Shrubsole64469032020-06-11 10:45:29 +02002704 // Change the degradation preference back. CPU underuse should not adapt since
2705 // QP is most limited.
Henrik Boström2671dac2020-05-19 16:29:09 +02002706 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002707 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
2708 video_source_.IncomingCapturedFrame(
2709 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2710 sink_.WaitForEncodedFrame(ntp_time);
2711 ntp_time += 100;
2712 // Resolution adaptations is gone after changing degradation preference.
2713 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2714 EXPECT_EQ(std::numeric_limits<int>::max(),
2715 video_source_.sink_wants().max_pixel_count);
2716 // The fps adaptation from above is now back.
2717 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2718
2719 // Trigger CPU underuse.
2720 video_stream_encoder_->TriggerCpuUnderuse();
2721 video_source_.IncomingCapturedFrame(
2722 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2723 sink_.WaitForEncodedFrame(ntp_time);
2724 ntp_time += kFrameIntervalMs;
Evan Shrubsole64469032020-06-11 10:45:29 +02002725 EXPECT_EQ(video_source_.sink_wants().max_framerate_fps, restricted_fps);
2726
2727 // Trigger QP underuse, fps should return to normal.
2728 video_stream_encoder_->TriggerQualityHigh();
2729 video_source_.IncomingCapturedFrame(
2730 CreateFrame(ntp_time, kFrameWidth, kFrameHeight));
2731 sink_.WaitForEncodedFrame(ntp_time);
2732 ntp_time += kFrameIntervalMs;
2733 EXPECT_THAT(video_source_.sink_wants(), FpsMax());
Evan Shrubsole2e2f6742020-05-14 10:41:15 +02002734
2735 video_stream_encoder_->Stop();
2736}
2737
mflodmancc3d4422017-08-03 08:27:51 -07002738TEST_F(VideoStreamEncoderTest, SinkWantsStoredByDegradationPreference) {
Henrik Boström381d1092020-05-12 18:49:07 +02002739 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002740 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002741 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
perkj803d97f2016-11-01 11:45:46 -07002742
sprangc5d62e22017-04-02 23:53:04 -07002743 const int kFrameWidth = 1280;
2744 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07002745
Åsa Persson8c1bf952018-09-13 10:42:19 +02002746 int64_t frame_timestamp = 1;
perkj803d97f2016-11-01 11:45:46 -07002747
kthelgason5e13d412016-12-01 03:59:51 -08002748 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002749 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002750 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002751 frame_timestamp += kFrameIntervalMs;
2752
perkj803d97f2016-11-01 11:45:46 -07002753 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002754 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002755 video_source_.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002756 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002757 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002758 frame_timestamp += kFrameIntervalMs;
sprang3ea3c772017-03-30 07:23:48 -07002759
asapersson0944a802017-04-07 00:57:58 -07002760 // Default degradation preference is maintain-framerate, so will lower max
sprangc5d62e22017-04-02 23:53:04 -07002761 // wanted resolution.
2762 EXPECT_FALSE(video_source_.sink_wants().target_pixel_count);
2763 EXPECT_LT(video_source_.sink_wants().max_pixel_count,
2764 kFrameWidth * kFrameHeight);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002765 EXPECT_EQ(kDefaultFramerate, video_source_.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002766
2767 // Set new source, switch to maintain-resolution.
perkj803d97f2016-11-01 11:45:46 -07002768 test::FrameForwarder new_video_source;
Henrik Boström381d1092020-05-12 18:49:07 +02002769 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002770 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002771 // Give the encoder queue time to process the change in degradation preference
2772 // by waiting for an encoded frame.
2773 new_video_source.IncomingCapturedFrame(
2774 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2775 sink_.WaitForEncodedFrame(frame_timestamp);
2776 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002777 // Initially no degradation registered.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002778 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002779
sprangc5d62e22017-04-02 23:53:04 -07002780 // Force an input frame rate to be available, or the adaptation call won't
2781 // know what framerate to adapt form.
asapersson02465b82017-04-10 01:12:52 -07002782 const int kInputFps = 30;
sprangc5d62e22017-04-02 23:53:04 -07002783 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson02465b82017-04-10 01:12:52 -07002784 stats.input_frame_rate = kInputFps;
sprangc5d62e22017-04-02 23:53:04 -07002785 stats_proxy_->SetMockStats(stats);
2786
mflodmancc3d4422017-08-03 08:27:51 -07002787 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07002788 new_video_source.IncomingCapturedFrame(
sprangc5d62e22017-04-02 23:53:04 -07002789 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002790 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002791 frame_timestamp += kFrameIntervalMs;
2792
2793 // Some framerate constraint should be set.
sprang84a37592017-02-10 07:04:27 -08002794 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
sprangc5d62e22017-04-02 23:53:04 -07002795 EXPECT_EQ(std::numeric_limits<int>::max(),
2796 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002797 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
sprangc5d62e22017-04-02 23:53:04 -07002798
asapersson02465b82017-04-10 01:12:52 -07002799 // Turn off degradation completely.
Henrik Boström381d1092020-05-12 18:49:07 +02002800 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
2801 &new_video_source, webrtc::DegradationPreference::DISABLED);
Henrik Boström07b17df2020-01-15 11:42:12 +01002802 // Give the encoder queue time to process the change in degradation preference
2803 // by waiting for an encoded frame.
2804 new_video_source.IncomingCapturedFrame(
2805 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2806 sink_.WaitForEncodedFrame(frame_timestamp);
2807 frame_timestamp += kFrameIntervalMs;
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002808 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
sprangc5d62e22017-04-02 23:53:04 -07002809
mflodmancc3d4422017-08-03 08:27:51 -07002810 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07002811 new_video_source.IncomingCapturedFrame(
2812 CreateFrame(frame_timestamp, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07002813 WaitForEncodedFrame(frame_timestamp);
sprangc5d62e22017-04-02 23:53:04 -07002814 frame_timestamp += kFrameIntervalMs;
2815
2816 // Still no degradation.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02002817 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
perkj803d97f2016-11-01 11:45:46 -07002818
2819 // Calling SetSource with resolution scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002820 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002821 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström07b17df2020-01-15 11:42:12 +01002822 // Give the encoder queue time to process the change in degradation preference
2823 // by waiting for an encoded frame.
2824 new_video_source.IncomingCapturedFrame(
2825 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2826 sink_.WaitForEncodedFrame(frame_timestamp);
2827 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002828 EXPECT_LT(new_video_source.sink_wants().max_pixel_count,
2829 kFrameWidth * kFrameHeight);
sprang84a37592017-02-10 07:04:27 -08002830 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
Åsa Persson8c1bf952018-09-13 10:42:19 +02002831 EXPECT_EQ(kDefaultFramerate, new_video_source.sink_wants().max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07002832
2833 // Calling SetSource with framerate scaling enabled apply the old SinkWants.
Henrik Boström381d1092020-05-12 18:49:07 +02002834 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002835 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01002836 // Give the encoder queue time to process the change in degradation preference
2837 // by waiting for an encoded frame.
2838 new_video_source.IncomingCapturedFrame(
2839 CreateFrame(frame_timestamp, kFrameWidth, kFrameWidth));
2840 sink_.WaitForEncodedFrame(frame_timestamp);
2841 frame_timestamp += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07002842 EXPECT_FALSE(new_video_source.sink_wants().target_pixel_count);
2843 EXPECT_EQ(std::numeric_limits<int>::max(),
2844 new_video_source.sink_wants().max_pixel_count);
asapersson02465b82017-04-10 01:12:52 -07002845 EXPECT_LT(new_video_source.sink_wants().max_framerate_fps, kInputFps);
perkj803d97f2016-11-01 11:45:46 -07002846
mflodmancc3d4422017-08-03 08:27:51 -07002847 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002848}
2849
mflodmancc3d4422017-08-03 08:27:51 -07002850TEST_F(VideoStreamEncoderTest, StatsTracksQualityAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002851 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002852 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07002853
asaperssonfab67072017-04-04 05:51:49 -07002854 const int kWidth = 1280;
2855 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002856 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002857 WaitForEncodedFrame(1);
asaperssonfab67072017-04-04 05:51:49 -07002858 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2859 EXPECT_FALSE(stats.bw_limited_resolution);
2860 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
2861
2862 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07002863 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07002864 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002865 WaitForEncodedFrame(2);
asaperssonfab67072017-04-04 05:51:49 -07002866
2867 stats = stats_proxy_->GetStats();
2868 EXPECT_TRUE(stats.bw_limited_resolution);
2869 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
2870
2871 // Trigger adapt up.
mflodmancc3d4422017-08-03 08:27:51 -07002872 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07002873 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002874 WaitForEncodedFrame(3);
asaperssonfab67072017-04-04 05:51:49 -07002875
2876 stats = stats_proxy_->GetStats();
2877 EXPECT_FALSE(stats.bw_limited_resolution);
2878 EXPECT_EQ(2, stats.number_of_quality_adapt_changes);
2879 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2880
mflodmancc3d4422017-08-03 08:27:51 -07002881 video_stream_encoder_->Stop();
asaperssonfab67072017-04-04 05:51:49 -07002882}
2883
mflodmancc3d4422017-08-03 08:27:51 -07002884TEST_F(VideoStreamEncoderTest, StatsTracksCpuAdaptationStats) {
Henrik Boström381d1092020-05-12 18:49:07 +02002885 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002886 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07002887
2888 const int kWidth = 1280;
2889 const int kHeight = 720;
asaperssonfab67072017-04-04 05:51:49 -07002890 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002891 WaitForEncodedFrame(1);
perkj803d97f2016-11-01 11:45:46 -07002892 VideoSendStream::Stats stats = stats_proxy_->GetStats();
2893 EXPECT_FALSE(stats.cpu_limited_resolution);
2894 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2895
2896 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002897 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002898 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002899 WaitForEncodedFrame(2);
perkj803d97f2016-11-01 11:45:46 -07002900
2901 stats = stats_proxy_->GetStats();
2902 EXPECT_TRUE(stats.cpu_limited_resolution);
2903 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2904
2905 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002906 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002907 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002908 WaitForEncodedFrame(3);
perkj803d97f2016-11-01 11:45:46 -07002909
2910 stats = stats_proxy_->GetStats();
2911 EXPECT_FALSE(stats.cpu_limited_resolution);
2912 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asaperssonfab67072017-04-04 05:51:49 -07002913 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07002914
mflodmancc3d4422017-08-03 08:27:51 -07002915 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07002916}
2917
mflodmancc3d4422017-08-03 08:27:51 -07002918TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsCpuAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002919 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002920 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002921
asaperssonfab67072017-04-04 05:51:49 -07002922 const int kWidth = 1280;
2923 const int kHeight = 720;
2924 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002925 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002926 VideoSendStream::Stats stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002927 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002928 EXPECT_FALSE(stats.cpu_limited_resolution);
2929 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
2930
asaperssonfab67072017-04-04 05:51:49 -07002931 // Trigger CPU overuse.
mflodmancc3d4422017-08-03 08:27:51 -07002932 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07002933 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002934 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08002935 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002936 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002937 EXPECT_TRUE(stats.cpu_limited_resolution);
2938 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2939
2940 // Set new source with adaptation still enabled.
2941 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07002942 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002943 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002944
asaperssonfab67072017-04-04 05:51:49 -07002945 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002946 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08002947 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002948 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002949 EXPECT_TRUE(stats.cpu_limited_resolution);
2950 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2951
2952 // Set adaptation disabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002953 video_stream_encoder_->SetSource(&new_video_source,
2954 webrtc::DegradationPreference::DISABLED);
kthelgason876222f2016-11-29 01:44:11 -08002955
asaperssonfab67072017-04-04 05:51:49 -07002956 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002957 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08002958 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002959 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002960 EXPECT_FALSE(stats.cpu_limited_resolution);
2961 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2962
2963 // Set adaptation back to enabled.
mflodmancc3d4422017-08-03 08:27:51 -07002964 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07002965 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
kthelgason876222f2016-11-29 01:44:11 -08002966
asaperssonfab67072017-04-04 05:51:49 -07002967 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002968 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08002969 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002970 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002971 EXPECT_TRUE(stats.cpu_limited_resolution);
2972 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
2973
asaperssonfab67072017-04-04 05:51:49 -07002974 // Trigger CPU normal use.
Henrik Boström91aa7322020-04-28 12:24:33 +02002975 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07002976 new_video_source.IncomingCapturedFrame(CreateFrame(6, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002977 WaitForEncodedFrame(6);
kthelgason876222f2016-11-29 01:44:11 -08002978 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07002979 EXPECT_FALSE(stats.bw_limited_resolution);
kthelgason876222f2016-11-29 01:44:11 -08002980 EXPECT_FALSE(stats.cpu_limited_resolution);
2981 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07002982 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002983
mflodmancc3d4422017-08-03 08:27:51 -07002984 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08002985}
2986
mflodmancc3d4422017-08-03 08:27:51 -07002987TEST_F(VideoStreamEncoderTest, SwitchingSourceKeepsQualityAdaptation) {
Henrik Boström381d1092020-05-12 18:49:07 +02002988 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02002989 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08002990
asaperssonfab67072017-04-04 05:51:49 -07002991 const int kWidth = 1280;
2992 const int kHeight = 720;
2993 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07002994 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08002995 VideoSendStream::Stats stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08002996 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07002997 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07002998 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08002999
3000 // Set new source with adaptation still enabled.
3001 test::FrameForwarder new_video_source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003002 video_stream_encoder_->SetSource(&new_video_source,
3003 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08003004
asaperssonfab67072017-04-04 05:51:49 -07003005 new_video_source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003006 WaitForEncodedFrame(2);
kthelgason876222f2016-11-29 01:44:11 -08003007 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003008 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003009 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003010 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003011
asaperssonfab67072017-04-04 05:51:49 -07003012 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003013 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003014 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003015 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003016 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003017 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003018 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003019 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003020
asaperssonfab67072017-04-04 05:51:49 -07003021 // Set new source with adaptation still enabled.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003022 video_stream_encoder_->SetSource(&new_video_source,
3023 webrtc::DegradationPreference::BALANCED);
kthelgason876222f2016-11-29 01:44:11 -08003024
asaperssonfab67072017-04-04 05:51:49 -07003025 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003026 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003027 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003028 EXPECT_TRUE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003029 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003030 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003031
asapersson02465b82017-04-10 01:12:52 -07003032 // Disable resolution scaling.
mflodmancc3d4422017-08-03 08:27:51 -07003033 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003034 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003035
asaperssonfab67072017-04-04 05:51:49 -07003036 new_video_source.IncomingCapturedFrame(CreateFrame(5, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003037 WaitForEncodedFrame(5);
kthelgason876222f2016-11-29 01:44:11 -08003038 stats = stats_proxy_->GetStats();
kthelgason876222f2016-11-29 01:44:11 -08003039 EXPECT_FALSE(stats.bw_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003040 EXPECT_FALSE(stats.bw_limited_framerate);
asaperssonfab67072017-04-04 05:51:49 -07003041 EXPECT_EQ(1, stats.number_of_quality_adapt_changes);
3042 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
kthelgason876222f2016-11-29 01:44:11 -08003043
mflodmancc3d4422017-08-03 08:27:51 -07003044 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003045}
3046
mflodmancc3d4422017-08-03 08:27:51 -07003047TEST_F(VideoStreamEncoderTest,
3048 QualityAdaptationStatsAreResetWhenScalerIsDisabled) {
Henrik Boström381d1092020-05-12 18:49:07 +02003049 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003050 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson36e9eb42017-03-31 05:29:12 -07003051
3052 const int kWidth = 1280;
3053 const int kHeight = 720;
Åsa Persson8c1bf952018-09-13 10:42:19 +02003054 int64_t timestamp_ms = kFrameIntervalMs;
asapersson36e9eb42017-03-31 05:29:12 -07003055 video_source_.set_adaptation_enabled(true);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003056 video_source_.IncomingCapturedFrame(
3057 CreateFrame(timestamp_ms, kWidth, kHeight));
3058 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003059 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3060 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3061 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3062
3063 // Trigger adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003064 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003065 timestamp_ms += kFrameIntervalMs;
3066 video_source_.IncomingCapturedFrame(
3067 CreateFrame(timestamp_ms, kWidth, kHeight));
3068 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003069 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3070 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3071 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3072
3073 // Trigger overuse.
mflodmancc3d4422017-08-03 08:27:51 -07003074 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003075 timestamp_ms += kFrameIntervalMs;
3076 video_source_.IncomingCapturedFrame(
3077 CreateFrame(timestamp_ms, kWidth, kHeight));
3078 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003079 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3080 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3081 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3082
Niels Möller4db138e2018-04-19 09:04:13 +02003083 // Leave source unchanged, but disable quality scaler.
asapersson36e9eb42017-03-31 05:29:12 -07003084 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02003085
3086 VideoEncoderConfig video_encoder_config;
3087 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
3088 // Make format different, to force recreation of encoder.
3089 video_encoder_config.video_format.parameters["foo"] = "foo";
3090 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02003091 kMaxPayloadLength);
Åsa Persson8c1bf952018-09-13 10:42:19 +02003092 timestamp_ms += kFrameIntervalMs;
3093 video_source_.IncomingCapturedFrame(
3094 CreateFrame(timestamp_ms, kWidth, kHeight));
3095 WaitForEncodedFrame(timestamp_ms);
asapersson36e9eb42017-03-31 05:29:12 -07003096 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3097 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3098 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3099
mflodmancc3d4422017-08-03 08:27:51 -07003100 video_stream_encoder_->Stop();
asapersson36e9eb42017-03-31 05:29:12 -07003101}
3102
mflodmancc3d4422017-08-03 08:27:51 -07003103TEST_F(VideoStreamEncoderTest,
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003104 StatsTracksCpuAdaptationStatsWhenSwitchingSource_Balanced) {
Henrik Boström381d1092020-05-12 18:49:07 +02003105 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003106 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003107
3108 const int kWidth = 1280;
3109 const int kHeight = 720;
3110 int sequence = 1;
3111
3112 // Enable BALANCED preference, no initial limitation.
3113 test::FrameForwarder source;
3114 video_stream_encoder_->SetSource(&source,
3115 webrtc::DegradationPreference::BALANCED);
3116 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3117 WaitForEncodedFrame(sequence++);
3118 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3119 EXPECT_FALSE(stats.cpu_limited_resolution);
3120 EXPECT_FALSE(stats.cpu_limited_framerate);
3121 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3122
3123 // Trigger CPU overuse, should now adapt down.
3124 video_stream_encoder_->TriggerCpuOveruse();
3125 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3126 WaitForEncodedFrame(sequence++);
3127 stats = stats_proxy_->GetStats();
3128 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3129
3130 // Set new degradation preference should clear restrictions since we changed
3131 // from BALANCED.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003132 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003133 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
3134 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3135 WaitForEncodedFrame(sequence++);
3136 stats = stats_proxy_->GetStats();
3137 EXPECT_FALSE(stats.cpu_limited_resolution);
3138 EXPECT_FALSE(stats.cpu_limited_framerate);
3139 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3140
3141 // Force an input frame rate to be available, or the adaptation call won't
3142 // know what framerate to adapt from.
3143 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3144 mock_stats.input_frame_rate = 30;
3145 stats_proxy_->SetMockStats(mock_stats);
3146 video_stream_encoder_->TriggerCpuOveruse();
3147 stats_proxy_->ResetMockStats();
3148 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3149 WaitForEncodedFrame(sequence++);
3150
3151 // We have now adapted once.
3152 stats = stats_proxy_->GetStats();
3153 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3154
3155 // Back to BALANCED, should clear the restrictions again.
Evan Shrubsolede2049e2020-05-25 16:54:21 +02003156 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
3157 &source, webrtc::DegradationPreference::BALANCED);
Evan Shrubsole33be9df2020-03-05 18:39:32 +01003158 source.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
3159 WaitForEncodedFrame(sequence++);
3160 stats = stats_proxy_->GetStats();
3161 EXPECT_FALSE(stats.cpu_limited_resolution);
3162 EXPECT_FALSE(stats.cpu_limited_framerate);
3163 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3164
3165 video_stream_encoder_->Stop();
3166}
3167
3168TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003169 StatsTracksCpuAdaptationStatsWhenSwitchingSource) {
Henrik Boström381d1092020-05-12 18:49:07 +02003170 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003171 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
perkj803d97f2016-11-01 11:45:46 -07003172
asapersson0944a802017-04-07 00:57:58 -07003173 const int kWidth = 1280;
3174 const int kHeight = 720;
sprang84a37592017-02-10 07:04:27 -08003175 int sequence = 1;
perkj803d97f2016-11-01 11:45:46 -07003176
asaperssonfab67072017-04-04 05:51:49 -07003177 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003178 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003179 VideoSendStream::Stats stats = stats_proxy_->GetStats();
sprang84a37592017-02-10 07:04:27 -08003180 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003181 EXPECT_FALSE(stats.cpu_limited_framerate);
sprang84a37592017-02-10 07:04:27 -08003182 EXPECT_EQ(0, stats.number_of_cpu_adapt_changes);
3183
asapersson02465b82017-04-10 01:12:52 -07003184 // Trigger CPU overuse, should now adapt down.
mflodmancc3d4422017-08-03 08:27:51 -07003185 video_stream_encoder_->TriggerCpuOveruse();
asaperssonfab67072017-04-04 05:51:49 -07003186 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003187 WaitForEncodedFrame(sequence++);
sprang84a37592017-02-10 07:04:27 -08003188 stats = stats_proxy_->GetStats();
perkj803d97f2016-11-01 11:45:46 -07003189 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003190 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003191 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3192
3193 // Set new source with adaptation still enabled.
3194 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003195 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003196 &new_video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
perkj803d97f2016-11-01 11:45:46 -07003197
3198 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003199 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003200 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003201 stats = stats_proxy_->GetStats();
3202 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003203 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003204 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3205
sprangc5d62e22017-04-02 23:53:04 -07003206 // Set cpu adaptation by frame dropping.
mflodmancc3d4422017-08-03 08:27:51 -07003207 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003208 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
perkj803d97f2016-11-01 11:45:46 -07003209 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003210 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003211 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003212 stats = stats_proxy_->GetStats();
sprangc5d62e22017-04-02 23:53:04 -07003213 // Not adapted at first.
perkj803d97f2016-11-01 11:45:46 -07003214 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson09f05612017-05-15 23:40:18 -07003215 EXPECT_FALSE(stats.cpu_limited_framerate);
perkj803d97f2016-11-01 11:45:46 -07003216 EXPECT_EQ(1, stats.number_of_cpu_adapt_changes);
3217
sprangc5d62e22017-04-02 23:53:04 -07003218 // Force an input frame rate to be available, or the adaptation call won't
asapersson09f05612017-05-15 23:40:18 -07003219 // know what framerate to adapt from.
sprangc5d62e22017-04-02 23:53:04 -07003220 VideoSendStream::Stats mock_stats = stats_proxy_->GetStats();
3221 mock_stats.input_frame_rate = 30;
3222 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003223 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003224 stats_proxy_->ResetMockStats();
3225
3226 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003227 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003228 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003229
3230 // Framerate now adapted.
3231 stats = stats_proxy_->GetStats();
asapersson09f05612017-05-15 23:40:18 -07003232 EXPECT_FALSE(stats.cpu_limited_resolution);
3233 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003234 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3235
3236 // Disable CPU adaptation.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003237 video_stream_encoder_->SetSource(&new_video_source,
3238 webrtc::DegradationPreference::DISABLED);
sprangc5d62e22017-04-02 23:53:04 -07003239 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003240 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003241 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003242
3243 stats = stats_proxy_->GetStats();
3244 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003245 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003246 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3247
3248 // Try to trigger overuse. Should not succeed.
3249 stats_proxy_->SetMockStats(mock_stats);
mflodmancc3d4422017-08-03 08:27:51 -07003250 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07003251 stats_proxy_->ResetMockStats();
3252
3253 stats = stats_proxy_->GetStats();
3254 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003255 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003256 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
3257
3258 // Switch back the source with resolution adaptation enabled.
mflodmancc3d4422017-08-03 08:27:51 -07003259 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003260 &video_source_, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssonfab67072017-04-04 05:51:49 -07003261 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003262 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003263 stats = stats_proxy_->GetStats();
3264 EXPECT_TRUE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003265 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003266 EXPECT_EQ(2, stats.number_of_cpu_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003267
3268 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003269 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonfab67072017-04-04 05:51:49 -07003270 video_source_.IncomingCapturedFrame(CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003271 WaitForEncodedFrame(sequence++);
perkj803d97f2016-11-01 11:45:46 -07003272 stats = stats_proxy_->GetStats();
3273 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003274 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003275 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3276
3277 // Back to the source with adaptation off, set it back to maintain-resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003278 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003279 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07003280 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003281 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003282 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003283 stats = stats_proxy_->GetStats();
asapersson13874762017-06-07 00:01:02 -07003284 // Disabled, since we previously switched the source to disabled.
sprangc5d62e22017-04-02 23:53:04 -07003285 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003286 EXPECT_TRUE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003287 EXPECT_EQ(3, stats.number_of_cpu_adapt_changes);
3288
3289 // Trigger CPU normal usage.
Henrik Boström91aa7322020-04-28 12:24:33 +02003290 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07003291 new_video_source.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07003292 CreateFrame(sequence, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003293 WaitForEncodedFrame(sequence++);
sprangc5d62e22017-04-02 23:53:04 -07003294 stats = stats_proxy_->GetStats();
3295 EXPECT_FALSE(stats.cpu_limited_resolution);
asapersson13874762017-06-07 00:01:02 -07003296 EXPECT_FALSE(stats.cpu_limited_framerate);
sprangc5d62e22017-04-02 23:53:04 -07003297 EXPECT_EQ(4, stats.number_of_cpu_adapt_changes);
asapersson02465b82017-04-10 01:12:52 -07003298 EXPECT_EQ(0, stats.number_of_quality_adapt_changes);
perkj803d97f2016-11-01 11:45:46 -07003299
mflodmancc3d4422017-08-03 08:27:51 -07003300 video_stream_encoder_->Stop();
perkj803d97f2016-11-01 11:45:46 -07003301}
3302
mflodmancc3d4422017-08-03 08:27:51 -07003303TEST_F(VideoStreamEncoderTest,
3304 ScalingUpAndDownDoesNothingWithMaintainResolution) {
asaperssonfab67072017-04-04 05:51:49 -07003305 const int kWidth = 1280;
3306 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003307 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003308 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason876222f2016-11-29 01:44:11 -08003309
asaperssonfab67072017-04-04 05:51:49 -07003310 // Expect no scaling to begin with.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003311 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
kthelgason876222f2016-11-29 01:44:11 -08003312
asaperssonfab67072017-04-04 05:51:49 -07003313 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003314 WaitForEncodedFrame(1);
kthelgason876222f2016-11-29 01:44:11 -08003315
asaperssonfab67072017-04-04 05:51:49 -07003316 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003317 video_stream_encoder_->TriggerQualityLow();
kthelgason5e13d412016-12-01 03:59:51 -08003318
asaperssonfab67072017-04-04 05:51:49 -07003319 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003320 WaitForEncodedFrame(2);
kthelgason5e13d412016-12-01 03:59:51 -08003321
kthelgason876222f2016-11-29 01:44:11 -08003322 // Expect a scale down.
3323 EXPECT_TRUE(video_source_.sink_wants().max_pixel_count);
asaperssonfab67072017-04-04 05:51:49 -07003324 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason876222f2016-11-29 01:44:11 -08003325
asapersson02465b82017-04-10 01:12:52 -07003326 // Set resolution scaling disabled.
kthelgason876222f2016-11-29 01:44:11 -08003327 test::FrameForwarder new_video_source;
mflodmancc3d4422017-08-03 08:27:51 -07003328 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003329 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason876222f2016-11-29 01:44:11 -08003330
asaperssonfab67072017-04-04 05:51:49 -07003331 // Trigger scale down.
mflodmancc3d4422017-08-03 08:27:51 -07003332 video_stream_encoder_->TriggerQualityLow();
asaperssonfab67072017-04-04 05:51:49 -07003333 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003334 WaitForEncodedFrame(3);
kthelgason876222f2016-11-29 01:44:11 -08003335
asaperssonfab67072017-04-04 05:51:49 -07003336 // Expect no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003337 EXPECT_EQ(std::numeric_limits<int>::max(),
3338 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003339
asaperssonfab67072017-04-04 05:51:49 -07003340 // Trigger scale up.
mflodmancc3d4422017-08-03 08:27:51 -07003341 video_stream_encoder_->TriggerQualityHigh();
asaperssonfab67072017-04-04 05:51:49 -07003342 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003343 WaitForEncodedFrame(4);
kthelgason876222f2016-11-29 01:44:11 -08003344
asapersson02465b82017-04-10 01:12:52 -07003345 // Expect nothing to change, still no scaling.
sprangc5d62e22017-04-02 23:53:04 -07003346 EXPECT_EQ(std::numeric_limits<int>::max(),
3347 new_video_source.sink_wants().max_pixel_count);
kthelgason876222f2016-11-29 01:44:11 -08003348
mflodmancc3d4422017-08-03 08:27:51 -07003349 video_stream_encoder_->Stop();
kthelgason876222f2016-11-29 01:44:11 -08003350}
3351
mflodmancc3d4422017-08-03 08:27:51 -07003352TEST_F(VideoStreamEncoderTest,
3353 SkipsSameAdaptDownRequest_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003354 const int kWidth = 1280;
3355 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003356 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003357 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003358
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003359 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003360 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003361 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003362 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003363
3364 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003365 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003366 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003367 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3368 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3369
3370 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003371 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003372 EXPECT_THAT(source.sink_wants(),
3373 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003374 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3375 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3376 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3377
3378 // Trigger adapt down for same input resolution, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003379 video_stream_encoder_->TriggerCpuOveruse();
asapersson02465b82017-04-10 01:12:52 -07003380 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3381 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3382 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3383
mflodmancc3d4422017-08-03 08:27:51 -07003384 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003385}
3386
mflodmancc3d4422017-08-03 08:27:51 -07003387TEST_F(VideoStreamEncoderTest, SkipsSameOrLargerAdaptDownRequest_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003388 const int kWidth = 1280;
3389 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003390 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003391 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003392
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003393 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003394 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003395 video_stream_encoder_->SetSource(&source,
3396 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003397 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3398 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003399 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003400
3401 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003402 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003403 EXPECT_THAT(source.sink_wants(),
3404 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003405 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3406 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3407 const int kLastMaxPixelCount = source.sink_wants().max_pixel_count;
3408
3409 // Trigger adapt down for same input resolution, expect no change.
3410 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3411 sink_.WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07003412 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003413 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3414 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3415 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3416
3417 // Trigger adapt down for larger input resolution, expect no change.
3418 source.IncomingCapturedFrame(CreateFrame(3, kWidth + 1, kHeight + 1));
3419 sink_.WaitForEncodedFrame(3);
mflodmancc3d4422017-08-03 08:27:51 -07003420 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07003421 EXPECT_EQ(kLastMaxPixelCount, source.sink_wants().max_pixel_count);
3422 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3423 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3424
mflodmancc3d4422017-08-03 08:27:51 -07003425 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003426}
3427
mflodmancc3d4422017-08-03 08:27:51 -07003428TEST_F(VideoStreamEncoderTest,
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003429 FpsCountReturnsToZeroForFewerAdaptationsUpThanDown) {
3430 const int kWidth = 640;
3431 const int kHeight = 360;
3432 const int64_t kFrameIntervalMs = 150;
3433 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003434 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003435
3436 // Enable BALANCED preference, no initial limitation.
3437 AdaptingFrameForwarder source(&time_controller_);
3438 source.set_adaptation_enabled(true);
3439 video_stream_encoder_->SetSource(&source,
3440 webrtc::DegradationPreference::BALANCED);
3441
3442 int64_t timestamp_ms = kFrameIntervalMs;
3443 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3444 sink_.WaitForEncodedFrame(kWidth, kHeight);
3445 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3446 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3447 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3448 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3449
3450 // Trigger adapt down, expect reduced fps (640x360@15fps).
3451 video_stream_encoder_->TriggerQualityLow();
3452 timestamp_ms += kFrameIntervalMs;
3453 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3454 sink_.WaitForEncodedFrame(timestamp_ms);
3455 EXPECT_THAT(source.sink_wants(),
3456 FpsMatchesResolutionMax(Lt(kDefaultFramerate)));
3457 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3458 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3459 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3460
3461 // Source requests 270p, expect reduced resolution (480x270@15fps).
3462 source.OnOutputFormatRequest(480, 270);
3463 timestamp_ms += kFrameIntervalMs;
3464 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3465 WaitForEncodedFrame(480, 270);
3466 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3467
3468 // Trigger adapt down, expect reduced fps (480x270@10fps).
3469 video_stream_encoder_->TriggerQualityLow();
3470 timestamp_ms += kFrameIntervalMs;
3471 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3472 sink_.WaitForEncodedFrame(timestamp_ms);
3473 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3474 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3475 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3476 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3477
3478 // Source requests QVGA, expect reduced resolution (320x180@10fps).
3479 source.OnOutputFormatRequest(320, 180);
3480 timestamp_ms += kFrameIntervalMs;
3481 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3482 WaitForEncodedFrame(320, 180);
3483 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3484
3485 // Trigger adapt down, expect reduced fps (320x180@7fps).
3486 video_stream_encoder_->TriggerQualityLow();
3487 timestamp_ms += kFrameIntervalMs;
3488 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3489 sink_.WaitForEncodedFrame(timestamp_ms);
3490 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3491 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3492 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3493 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3494
3495 // Source requests VGA, expect increased resolution (640x360@7fps).
3496 source.OnOutputFormatRequest(640, 360);
3497 timestamp_ms += kFrameIntervalMs;
3498 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3499 WaitForEncodedFrame(timestamp_ms);
3500 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3501
3502 // Trigger adapt up, expect increased fps (640x360@(max-2)fps).
3503 video_stream_encoder_->TriggerQualityHigh();
3504 timestamp_ms += kFrameIntervalMs;
3505 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3506 WaitForEncodedFrame(timestamp_ms);
3507 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3508 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3509 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3510 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3511
3512 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3513 video_stream_encoder_->TriggerQualityHigh();
3514 timestamp_ms += kFrameIntervalMs;
3515 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3516 WaitForEncodedFrame(timestamp_ms);
3517 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3518 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3519 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3520 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3521
3522 // Trigger adapt up, expect increased fps (640x360@maxfps).
3523 video_stream_encoder_->TriggerQualityHigh();
3524 timestamp_ms += kFrameIntervalMs;
3525 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3526 WaitForEncodedFrame(timestamp_ms);
3527 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3528 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3529 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3530 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3531
3532 video_stream_encoder_->Stop();
3533}
3534
3535TEST_F(VideoStreamEncoderTest,
3536 FpsCountReturnsToZeroForFewerAdaptationsUpThanDownWithTwoResources) {
3537 const int kWidth = 1280;
3538 const int kHeight = 720;
3539 const int64_t kFrameIntervalMs = 150;
3540 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003541 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02003542
3543 // Enable BALANCED preference, no initial limitation.
3544 AdaptingFrameForwarder source(&time_controller_);
3545 source.set_adaptation_enabled(true);
3546 video_stream_encoder_->SetSource(&source,
3547 webrtc::DegradationPreference::BALANCED);
3548
3549 int64_t timestamp_ms = kFrameIntervalMs;
3550 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3551 sink_.WaitForEncodedFrame(kWidth, kHeight);
3552 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
3553 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3554 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3555 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3556
3557 // Trigger adapt down, expect scaled down resolution (960x540@maxfps).
3558 video_stream_encoder_->TriggerQualityLow();
3559 timestamp_ms += kFrameIntervalMs;
3560 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3561 sink_.WaitForEncodedFrame(timestamp_ms);
3562 EXPECT_THAT(source.sink_wants(),
3563 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
3564 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3565 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3566 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3567
3568 // Trigger adapt down, expect scaled down resolution (640x360@maxfps).
3569 video_stream_encoder_->TriggerQualityLow();
3570 timestamp_ms += kFrameIntervalMs;
3571 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3572 sink_.WaitForEncodedFrame(timestamp_ms);
3573 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
3574 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3575 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3576 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3577
3578 // Trigger adapt down, expect reduced fps (640x360@15fps).
3579 video_stream_encoder_->TriggerQualityLow();
3580 timestamp_ms += kFrameIntervalMs;
3581 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3582 WaitForEncodedFrame(timestamp_ms);
3583 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3584 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3585 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3586 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3587
3588 // Source requests QVGA, expect reduced resolution (320x180@15fps).
3589 source.OnOutputFormatRequest(320, 180);
3590 timestamp_ms += kFrameIntervalMs;
3591 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3592 WaitForEncodedFrame(320, 180);
3593 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3594 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3595
3596 // Trigger adapt down, expect reduced fps (320x180@7fps).
3597 video_stream_encoder_->TriggerCpuOveruse();
3598 timestamp_ms += kFrameIntervalMs;
3599 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3600 WaitForEncodedFrame(timestamp_ms);
3601 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
3602 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3603 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3604 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3605 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3606 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3607 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3608
3609 // Source requests HD, expect increased resolution (640x360@7fps).
3610 source.OnOutputFormatRequest(1280, 720);
3611 timestamp_ms += kFrameIntervalMs;
3612 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3613 WaitForEncodedFrame(timestamp_ms);
3614 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3615 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3616
3617 // Trigger adapt up, expect increased fps (640x360@(max-1)fps).
3618 video_stream_encoder_->TriggerCpuUnderuse();
3619 timestamp_ms += kFrameIntervalMs;
3620 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3621 WaitForEncodedFrame(timestamp_ms);
3622 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3623 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3624 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
3625 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3626 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
3627 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3628 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3629
3630 // Trigger adapt up, expect increased fps (640x360@maxfps).
3631 video_stream_encoder_->TriggerQualityHigh();
3632 video_stream_encoder_->TriggerCpuUnderuse();
3633 timestamp_ms += kFrameIntervalMs;
3634 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3635 WaitForEncodedFrame(timestamp_ms);
3636 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
3637 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3638 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3639 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3640 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3641 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3642 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3643
3644 // Trigger adapt up, expect increased resolution (960x570@maxfps).
3645 video_stream_encoder_->TriggerQualityHigh();
3646 video_stream_encoder_->TriggerCpuUnderuse();
3647 timestamp_ms += kFrameIntervalMs;
3648 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3649 WaitForEncodedFrame(timestamp_ms);
3650 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3651 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3652 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3653 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3654 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3655 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3656 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3657
3658 // Trigger adapt up, expect increased resolution (1280x720@maxfps).
3659 video_stream_encoder_->TriggerQualityHigh();
3660 video_stream_encoder_->TriggerCpuUnderuse();
3661 timestamp_ms += kFrameIntervalMs;
3662 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3663 WaitForEncodedFrame(timestamp_ms);
3664 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
3665 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3666 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
3667 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3668 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3669 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3670 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3671
3672 video_stream_encoder_->Stop();
3673}
3674
3675TEST_F(VideoStreamEncoderTest,
mflodmancc3d4422017-08-03 08:27:51 -07003676 NoChangeForInitialNormalUsage_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003677 const int kWidth = 1280;
3678 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003679 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003680 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003681
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003682 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003683 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003684 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003685 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003686
3687 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003688 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003689 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003690 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3691 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3692
3693 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003694 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003695 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003696 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3697 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3698
mflodmancc3d4422017-08-03 08:27:51 -07003699 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003700}
3701
mflodmancc3d4422017-08-03 08:27:51 -07003702TEST_F(VideoStreamEncoderTest,
3703 NoChangeForInitialNormalUsage_MaintainResolutionMode) {
asapersson02465b82017-04-10 01:12:52 -07003704 const int kWidth = 1280;
3705 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003706 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003707 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003708
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003709 // Enable MAINTAIN_RESOLUTION preference, no initial limitation.
asapersson02465b82017-04-10 01:12:52 -07003710 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07003711 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003712 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
asapersson02465b82017-04-10 01:12:52 -07003713
3714 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003715 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003716 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003717 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003718 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3719
3720 // Trigger adapt up, expect no change.
Henrik Boström91aa7322020-04-28 12:24:33 +02003721 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003722 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003723 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
asapersson02465b82017-04-10 01:12:52 -07003724 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3725
mflodmancc3d4422017-08-03 08:27:51 -07003726 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003727}
3728
mflodmancc3d4422017-08-03 08:27:51 -07003729TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07003730 const int kWidth = 1280;
3731 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003732 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003733 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003734
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003735 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07003736 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003737 video_stream_encoder_->SetSource(&source,
3738 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003739
3740 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3741 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003742 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003743 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3744 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3745 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3746
3747 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003748 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003749 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003750 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3751 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3752 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3753
mflodmancc3d4422017-08-03 08:27:51 -07003754 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07003755}
3756
mflodmancc3d4422017-08-03 08:27:51 -07003757TEST_F(VideoStreamEncoderTest, NoChangeForInitialNormalUsage_DisabledMode) {
asapersson09f05612017-05-15 23:40:18 -07003758 const int kWidth = 1280;
3759 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003760 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003761 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003762
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003763 // Enable DISABLED preference, no initial limitation.
asapersson09f05612017-05-15 23:40:18 -07003764 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003765 video_stream_encoder_->SetSource(&source,
3766 webrtc::DegradationPreference::DISABLED);
asapersson09f05612017-05-15 23:40:18 -07003767
3768 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3769 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003770 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003771 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3772 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3773 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3774
3775 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07003776 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003777 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003778 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3779 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
3780 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3781
mflodmancc3d4422017-08-03 08:27:51 -07003782 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003783}
3784
mflodmancc3d4422017-08-03 08:27:51 -07003785TEST_F(VideoStreamEncoderTest,
3786 AdaptsResolutionForLowQuality_MaintainFramerateMode) {
asapersson02465b82017-04-10 01:12:52 -07003787 const int kWidth = 1280;
3788 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003789 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003790 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson02465b82017-04-10 01:12:52 -07003791
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003792 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003793 AdaptingFrameForwarder source(&time_controller_);
asapersson02465b82017-04-10 01:12:52 -07003794 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003795 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003796 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asapersson02465b82017-04-10 01:12:52 -07003797
3798 source.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003799 WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003800 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003801 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3802 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3803
3804 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003805 video_stream_encoder_->TriggerQualityLow();
asapersson02465b82017-04-10 01:12:52 -07003806 source.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003807 WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003808 EXPECT_THAT(source.sink_wants(),
3809 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson02465b82017-04-10 01:12:52 -07003810 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3811 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3812
3813 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003814 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003815 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asapersson02465b82017-04-10 01:12:52 -07003816 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3817 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3818 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3819
mflodmancc3d4422017-08-03 08:27:51 -07003820 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07003821}
3822
mflodmancc3d4422017-08-03 08:27:51 -07003823TEST_F(VideoStreamEncoderTest,
3824 AdaptsFramerateForLowQuality_MaintainResolutionMode) {
asapersson09f05612017-05-15 23:40:18 -07003825 const int kWidth = 1280;
3826 const int kHeight = 720;
3827 const int kInputFps = 30;
Henrik Boström381d1092020-05-12 18:49:07 +02003828 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003829 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07003830
3831 VideoSendStream::Stats stats = stats_proxy_->GetStats();
3832 stats.input_frame_rate = kInputFps;
3833 stats_proxy_->SetMockStats(stats);
3834
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003835 // Expect no scaling to begin with (preference: MAINTAIN_FRAMERATE).
asapersson09f05612017-05-15 23:40:18 -07003836 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
3837 sink_.WaitForEncodedFrame(1);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003838 EXPECT_THAT(video_source_.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003839
3840 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003841 video_stream_encoder_->TriggerQualityLow();
asapersson09f05612017-05-15 23:40:18 -07003842 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
3843 sink_.WaitForEncodedFrame(2);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003844 EXPECT_THAT(video_source_.sink_wants(),
3845 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asapersson09f05612017-05-15 23:40:18 -07003846
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003847 // Enable MAINTAIN_RESOLUTION preference.
asapersson09f05612017-05-15 23:40:18 -07003848 test::FrameForwarder new_video_source;
Henrik Boström2671dac2020-05-19 16:29:09 +02003849 video_stream_encoder_->SetSourceAndWaitForRestrictionsUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003850 &new_video_source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Henrik Boström07b17df2020-01-15 11:42:12 +01003851 // Give the encoder queue time to process the change in degradation preference
3852 // by waiting for an encoded frame.
3853 new_video_source.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
3854 sink_.WaitForEncodedFrame(3);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003855 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003856
3857 // Trigger adapt down, expect reduced framerate.
mflodmancc3d4422017-08-03 08:27:51 -07003858 video_stream_encoder_->TriggerQualityLow();
Henrik Boström07b17df2020-01-15 11:42:12 +01003859 new_video_source.IncomingCapturedFrame(CreateFrame(4, kWidth, kHeight));
3860 sink_.WaitForEncodedFrame(4);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003861 EXPECT_THAT(new_video_source.sink_wants(),
3862 FpsMatchesResolutionMax(Lt(kInputFps)));
asapersson09f05612017-05-15 23:40:18 -07003863
3864 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07003865 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003866 EXPECT_THAT(new_video_source.sink_wants(), FpsMaxResolutionMax());
asapersson09f05612017-05-15 23:40:18 -07003867
mflodmancc3d4422017-08-03 08:27:51 -07003868 video_stream_encoder_->Stop();
asapersson09f05612017-05-15 23:40:18 -07003869}
3870
mflodmancc3d4422017-08-03 08:27:51 -07003871TEST_F(VideoStreamEncoderTest, DoesNotScaleBelowSetResolutionLimit) {
asaperssond0de2952017-04-21 01:47:31 -07003872 const int kWidth = 1280;
3873 const int kHeight = 720;
3874 const size_t kNumFrames = 10;
3875
Henrik Boström381d1092020-05-12 18:49:07 +02003876 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003877 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003878
asaperssond0de2952017-04-21 01:47:31 -07003879 // Enable adapter, expected input resolutions when downscaling:
asapersson142fcc92017-08-17 08:58:54 -07003880 // 1280x720 -> 960x540 -> 640x360 -> 480x270 -> 320x180 (kMinPixelsPerFrame)
asaperssond0de2952017-04-21 01:47:31 -07003881 video_source_.set_adaptation_enabled(true);
3882
3883 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3884 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3885
3886 int downscales = 0;
3887 for (size_t i = 1; i <= kNumFrames; i++) {
Åsa Persson8c1bf952018-09-13 10:42:19 +02003888 video_source_.IncomingCapturedFrame(
3889 CreateFrame(i * kFrameIntervalMs, kWidth, kHeight));
3890 WaitForEncodedFrame(i * kFrameIntervalMs);
asaperssond0de2952017-04-21 01:47:31 -07003891
asaperssonfab67072017-04-04 05:51:49 -07003892 // Trigger scale down.
asaperssond0de2952017-04-21 01:47:31 -07003893 rtc::VideoSinkWants last_wants = video_source_.sink_wants();
mflodmancc3d4422017-08-03 08:27:51 -07003894 video_stream_encoder_->TriggerQualityLow();
sprangc5d62e22017-04-02 23:53:04 -07003895 EXPECT_GE(video_source_.sink_wants().max_pixel_count, kMinPixelsPerFrame);
asaperssond0de2952017-04-21 01:47:31 -07003896
3897 if (video_source_.sink_wants().max_pixel_count < last_wants.max_pixel_count)
3898 ++downscales;
3899
3900 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3901 EXPECT_EQ(downscales,
3902 stats_proxy_->GetStats().number_of_quality_adapt_changes);
3903 EXPECT_GT(downscales, 0);
kthelgason5e13d412016-12-01 03:59:51 -08003904 }
mflodmancc3d4422017-08-03 08:27:51 -07003905 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003906}
3907
mflodmancc3d4422017-08-03 08:27:51 -07003908TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07003909 AdaptsResolutionUpAndDownTwiceOnOveruse_MaintainFramerateMode) {
3910 const int kWidth = 1280;
3911 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003912 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003913 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07003914
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003915 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003916 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07003917 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07003918 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003919 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07003920
Åsa Persson8c1bf952018-09-13 10:42:19 +02003921 int64_t timestamp_ms = kFrameIntervalMs;
3922 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003923 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003924 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003925 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3926 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3927
3928 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003929 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003930 timestamp_ms += kFrameIntervalMs;
3931 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3932 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003933 EXPECT_THAT(source.sink_wants(),
3934 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003935 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3936 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3937
3938 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003939 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003940 timestamp_ms += kFrameIntervalMs;
3941 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07003942 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003943 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003944 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3945 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3946
3947 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003948 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003949 timestamp_ms += kFrameIntervalMs;
3950 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3951 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003952 EXPECT_THAT(source.sink_wants(),
3953 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07003954 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
3955 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3956
3957 // Trigger adapt up, expect no restriction.
Henrik Boström91aa7322020-04-28 12:24:33 +02003958 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003959 timestamp_ms += kFrameIntervalMs;
3960 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asapersson09f05612017-05-15 23:40:18 -07003961 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003962 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07003963 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
3964 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
3965
mflodmancc3d4422017-08-03 08:27:51 -07003966 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07003967}
3968
mflodmancc3d4422017-08-03 08:27:51 -07003969TEST_F(VideoStreamEncoderTest,
asaperssonf7e294d2017-06-13 23:25:22 -07003970 AdaptsResolutionUpAndDownTwiceForLowQuality_BalancedMode_NoFpsLimit) {
3971 const int kWidth = 1280;
3972 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02003973 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02003974 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07003975
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003976 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02003977 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07003978 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07003979 video_stream_encoder_->SetSource(&source,
3980 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07003981
Åsa Persson8c1bf952018-09-13 10:42:19 +02003982 int64_t timestamp_ms = kFrameIntervalMs;
3983 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07003984 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003985 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07003986 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
3987 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3988
3989 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07003990 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02003991 timestamp_ms += kFrameIntervalMs;
3992 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
3993 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02003994 EXPECT_THAT(source.sink_wants(),
3995 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07003996 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
3997 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
3998
3999 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07004000 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004001 timestamp_ms += kFrameIntervalMs;
4002 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07004003 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004004 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004005 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4006 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4007
4008 // Trigger adapt down, expect scaled down resolution.
mflodmancc3d4422017-08-03 08:27:51 -07004009 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004010 timestamp_ms += kFrameIntervalMs;
4011 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4012 sink_.WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004013 EXPECT_THAT(source.sink_wants(),
4014 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07004015 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4016 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4017
4018 // Trigger adapt up, expect no restriction.
mflodmancc3d4422017-08-03 08:27:51 -07004019 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004020 timestamp_ms += kFrameIntervalMs;
4021 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
asaperssonf7e294d2017-06-13 23:25:22 -07004022 sink_.WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004023 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07004024 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4025 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4026
mflodmancc3d4422017-08-03 08:27:51 -07004027 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07004028}
4029
Sergey Silkin41c650b2019-10-14 13:12:19 +02004030TEST_F(VideoStreamEncoderTest, AdaptUpIfBwEstimateIsHigherThanMinBitrate) {
4031 fake_encoder_.SetResolutionBitrateLimits(
4032 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4033
Henrik Boström381d1092020-05-12 18:49:07 +02004034 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004035 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4036 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4037 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4038 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004039
4040 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004041 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004042 source.set_adaptation_enabled(true);
4043 video_stream_encoder_->SetSource(
4044 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4045
4046 // Insert 720p frame.
4047 int64_t timestamp_ms = kFrameIntervalMs;
4048 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4049 WaitForEncodedFrame(1280, 720);
4050
4051 // Reduce bitrate and trigger adapt down.
Henrik Boström381d1092020-05-12 18:49:07 +02004052 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004053 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4054 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4055 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4056 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004057 video_stream_encoder_->TriggerQualityLow();
4058
4059 // Insert 720p frame. It should be downscaled and encoded.
4060 timestamp_ms += kFrameIntervalMs;
4061 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4062 WaitForEncodedFrame(960, 540);
4063
4064 // Trigger adapt up. Higher resolution should not be requested duo to lack
4065 // of bitrate.
4066 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004067 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMatches(Lt(1280 * 720)));
Sergey Silkin41c650b2019-10-14 13:12:19 +02004068
4069 // Increase bitrate.
Henrik Boström381d1092020-05-12 18:49:07 +02004070 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004071 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4072 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps),
4073 DataRate::BitsPerSec(kEncoderBitrateLimits720p.min_start_bitrate_bps), 0,
4074 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004075
4076 // Trigger adapt up. Higher resolution should be requested.
4077 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004078 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Sergey Silkin41c650b2019-10-14 13:12:19 +02004079
4080 video_stream_encoder_->Stop();
4081}
4082
4083TEST_F(VideoStreamEncoderTest, DropFirstFramesIfBwEstimateIsTooLow) {
4084 fake_encoder_.SetResolutionBitrateLimits(
4085 {kEncoderBitrateLimits540p, kEncoderBitrateLimits720p});
4086
4087 // Set bitrate equal to min bitrate of 540p.
Henrik Boström381d1092020-05-12 18:49:07 +02004088 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01004089 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4090 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps),
4091 DataRate::BitsPerSec(kEncoderBitrateLimits540p.min_start_bitrate_bps), 0,
4092 0, 0);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004093
4094 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004095 AdaptingFrameForwarder source(&time_controller_);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004096 source.set_adaptation_enabled(true);
4097 video_stream_encoder_->SetSource(
4098 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
4099
4100 // Insert 720p frame. It should be dropped and lower resolution should be
4101 // requested.
4102 int64_t timestamp_ms = kFrameIntervalMs;
4103 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4104 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02004105 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < 1280 * 720, 5000);
Sergey Silkin41c650b2019-10-14 13:12:19 +02004106
4107 // Insert 720p frame. It should be downscaled and encoded.
4108 timestamp_ms += kFrameIntervalMs;
4109 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
4110 WaitForEncodedFrame(960, 540);
4111
4112 video_stream_encoder_->Stop();
4113}
4114
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004115class BalancedDegradationTest : public VideoStreamEncoderTest {
4116 protected:
4117 void SetupTest() {
4118 // Reset encoder for field trials to take effect.
4119 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02004120 OnBitrateUpdated(kTargetBitrate);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004121
4122 // Enable BALANCED preference.
4123 source_.set_adaptation_enabled(true);
Åsa Perssonccfb3402019-09-25 15:13:04 +02004124 video_stream_encoder_->SetSource(&source_, DegradationPreference::BALANCED);
4125 }
4126
Asa Persson606d3cb2021-10-04 10:07:11 +02004127 void OnBitrateUpdated(DataRate bitrate) {
Henrik Boström381d1092020-05-12 18:49:07 +02004128 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004129 bitrate, bitrate, bitrate, 0, 0, 0);
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004130 }
4131
Åsa Persson45b176f2019-09-30 11:19:05 +02004132 void InsertFrame() {
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004133 timestamp_ms_ += kFrameIntervalMs;
4134 source_.IncomingCapturedFrame(CreateFrame(timestamp_ms_, kWidth, kHeight));
Åsa Persson45b176f2019-09-30 11:19:05 +02004135 }
4136
4137 void InsertFrameAndWaitForEncoded() {
4138 InsertFrame();
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004139 sink_.WaitForEncodedFrame(timestamp_ms_);
4140 }
4141
4142 const int kWidth = 640; // pixels:640x360=230400
4143 const int kHeight = 360;
4144 const int64_t kFrameIntervalMs = 150; // Use low fps to not drop any frame.
4145 int64_t timestamp_ms_ = 0;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004146 AdaptingFrameForwarder source_{&time_controller_};
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004147};
4148
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004149TEST_F(BalancedDegradationTest, AdaptDownTwiceIfMinFpsDiffLtThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004150 test::ScopedKeyValueConfig field_trials(
4151 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004152 "WebRTC-Video-BalancedDegradationSettings/"
4153 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4154 SetupTest();
4155
4156 // Force input frame rate.
4157 const int kInputFps = 24;
4158 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4159 stats.input_frame_rate = kInputFps;
4160 stats_proxy_->SetMockStats(stats);
4161
Åsa Persson45b176f2019-09-30 11:19:05 +02004162 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004163 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004164
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004165 // Trigger adapt down, expect scaled down framerate and resolution,
4166 // since Fps diff (input-requested:0) < threshold.
4167 video_stream_encoder_->TriggerQualityLow();
4168 EXPECT_THAT(source_.sink_wants(),
4169 AllOf(WantsFps(Eq(24)), WantsMaxPixels(Le(230400))));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004170
4171 video_stream_encoder_->Stop();
4172}
4173
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004174TEST_F(BalancedDegradationTest, AdaptDownOnceIfFpsDiffGeThreshold) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004175 test::ScopedKeyValueConfig field_trials(
4176 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004177 "WebRTC-Video-BalancedDegradationSettings/"
4178 "pixels:57600|129600|230400,fps:7|10|24,fps_diff:1|1|1/");
4179 SetupTest();
4180
4181 // Force input frame rate.
4182 const int kInputFps = 25;
4183 VideoSendStream::Stats stats = stats_proxy_->GetStats();
4184 stats.input_frame_rate = kInputFps;
4185 stats_proxy_->SetMockStats(stats);
4186
Åsa Persson45b176f2019-09-30 11:19:05 +02004187 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004188 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004189
Evan Shrubsolea1c77f62020-08-10 11:01:06 +02004190 // Trigger adapt down, expect scaled down framerate only (640x360@24fps).
4191 // Fps diff (input-requested:1) == threshold.
4192 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004193 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(24)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004194
4195 video_stream_encoder_->Stop();
4196}
4197
4198TEST_F(BalancedDegradationTest, AdaptDownUsesCodecSpecificFps) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004199 test::ScopedKeyValueConfig field_trials(
4200 field_trials_,
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004201 "WebRTC-Video-BalancedDegradationSettings/"
4202 "pixels:57600|129600|230400,fps:7|10|24,vp8_fps:8|11|22/");
4203 SetupTest();
4204
4205 EXPECT_EQ(kVideoCodecVP8, video_encoder_config_.codec_type);
4206
Åsa Persson45b176f2019-09-30 11:19:05 +02004207 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004208 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004209
4210 // Trigger adapt down, expect scaled down framerate (640x360@22fps).
4211 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004212 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(22)));
Åsa Perssonb67c44c2019-09-24 15:25:32 +02004213
4214 video_stream_encoder_->Stop();
4215}
4216
Åsa Perssonccfb3402019-09-25 15:13:04 +02004217TEST_F(BalancedDegradationTest, NoAdaptUpIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004218 test::ScopedKeyValueConfig field_trials(
4219 field_trials_,
Åsa Persson1b247f12019-08-14 17:26:39 +02004220 "WebRTC-Video-BalancedDegradationSettings/"
4221 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004222 SetupTest();
Åsa Persson1b247f12019-08-14 17:26:39 +02004223
Asa Persson606d3cb2021-10-04 10:07:11 +02004224 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4225 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4226 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004227
Åsa Persson45b176f2019-09-30 11:19:05 +02004228 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004229 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson1b247f12019-08-14 17:26:39 +02004230 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4231
4232 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4233 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004234 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004235 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson1b247f12019-08-14 17:26:39 +02004236 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4237
4238 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4239 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004240 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004241 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004242 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4243
Åsa Persson30ab0152019-08-27 12:22:33 +02004244 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4245 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004246 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004247 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Perssonccfb3402019-09-25 15:13:04 +02004248 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 10);
Åsa Persson30ab0152019-08-27 12:22:33 +02004249 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4250
4251 // Trigger adapt up, expect no upscale in fps (target bitrate < min bitrate).
Åsa Persson1b247f12019-08-14 17:26:39 +02004252 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004253 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004254 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Åsa Persson1b247f12019-08-14 17:26:39 +02004255
Åsa Persson30ab0152019-08-27 12:22:33 +02004256 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004257 OnBitrateUpdated(kMinBitrate);
Åsa Persson1b247f12019-08-14 17:26:39 +02004258 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004259 InsertFrameAndWaitForEncoded();
Åsa Perssonccfb3402019-09-25 15:13:04 +02004260 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 14);
Åsa Persson30ab0152019-08-27 12:22:33 +02004261 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4262
4263 video_stream_encoder_->Stop();
4264}
4265
Åsa Perssonccfb3402019-09-25 15:13:04 +02004266TEST_F(BalancedDegradationTest,
Åsa Persson45b176f2019-09-30 11:19:05 +02004267 InitialFrameDropAdaptsFpsAndResolutionInOneStep) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004268 test::ScopedKeyValueConfig field_trials(
4269 field_trials_,
Åsa Persson45b176f2019-09-30 11:19:05 +02004270 "WebRTC-Video-BalancedDegradationSettings/"
4271 "pixels:57600|129600|230400,fps:7|24|24/");
4272 SetupTest();
Asa Persson606d3cb2021-10-04 10:07:11 +02004273 OnBitrateUpdated(kLowTargetBitrate);
Åsa Persson45b176f2019-09-30 11:19:05 +02004274
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004275 EXPECT_THAT(source_.sink_wants(), UnlimitedSinkWants());
Åsa Persson45b176f2019-09-30 11:19:05 +02004276
4277 // Insert frame, expect scaled down:
4278 // framerate (640x360@24fps) -> resolution (480x270@24fps).
4279 InsertFrame();
4280 EXPECT_FALSE(WaitForFrame(1000));
4281 EXPECT_LT(source_.sink_wants().max_pixel_count, kWidth * kHeight);
4282 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4283
4284 // Insert frame, expect scaled down:
4285 // resolution (320x180@24fps).
4286 InsertFrame();
4287 EXPECT_FALSE(WaitForFrame(1000));
4288 EXPECT_LT(source_.sink_wants().max_pixel_count,
4289 source_.last_wants().max_pixel_count);
4290 EXPECT_EQ(source_.sink_wants().max_framerate_fps, 24);
4291
4292 // Frame should not be dropped (min pixels per frame reached).
4293 InsertFrameAndWaitForEncoded();
4294
4295 video_stream_encoder_->Stop();
4296}
4297
4298TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004299 NoAdaptUpInResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004300 test::ScopedKeyValueConfig field_trials(
4301 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004302 "WebRTC-Video-BalancedDegradationSettings/"
4303 "pixels:57600|129600|230400,fps:7|10|14,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004304 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004305
Asa Persson606d3cb2021-10-04 10:07:11 +02004306 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4307 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4308 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004309
Åsa Persson45b176f2019-09-30 11:19:05 +02004310 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004311 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004312 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4313
4314 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4315 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004316 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004317 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004318 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4319
4320 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4321 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004322 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004323 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004324 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4325
4326 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4327 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004328 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004329 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson1b247f12019-08-14 17:26:39 +02004330 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4331
Åsa Persson30ab0152019-08-27 12:22:33 +02004332 // Trigger adapt up, expect upscaled fps (no bitrate limit) (480x270@14fps).
4333 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004334 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004335 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004336 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4337
4338 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
4339 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004340 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004341 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4342
4343 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004344 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004345 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004346 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004347 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004348 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4349
4350 video_stream_encoder_->Stop();
4351}
4352
Åsa Perssonccfb3402019-09-25 15:13:04 +02004353TEST_F(BalancedDegradationTest,
Åsa Persson30ab0152019-08-27 12:22:33 +02004354 NoAdaptUpInFpsAndResolutionIfBwEstimateIsLessThanMinBitrate) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01004355 test::ScopedKeyValueConfig field_trials(
4356 field_trials_,
Åsa Persson30ab0152019-08-27 12:22:33 +02004357 "WebRTC-Video-BalancedDegradationSettings/"
4358 "pixels:57600|129600|230400,fps:7|10|14,kbps:0|0|425,kbps_res:0|0|435/");
Åsa Perssonccfb3402019-09-25 15:13:04 +02004359 SetupTest();
Åsa Persson30ab0152019-08-27 12:22:33 +02004360
Asa Persson606d3cb2021-10-04 10:07:11 +02004361 const DataRate kMinBitrate = DataRate::KilobitsPerSec(425);
4362 const DataRate kTooLowMinBitrate = DataRate::KilobitsPerSec(424);
4363 const DataRate kResolutionMinBitrate = DataRate::KilobitsPerSec(435);
4364 const DataRate kTooLowMinResolutionBitrate = DataRate::KilobitsPerSec(434);
4365 OnBitrateUpdated(kTooLowMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004366
Åsa Persson45b176f2019-09-30 11:19:05 +02004367 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004368 EXPECT_THAT(source_.sink_wants(), FpsMaxResolutionMax());
Åsa Persson30ab0152019-08-27 12:22:33 +02004369 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4370
4371 // Trigger adapt down, expect scaled down framerate (640x360@14fps).
4372 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004373 InsertFrameAndWaitForEncoded();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004374 EXPECT_THAT(source_.sink_wants(), FpsMatchesResolutionMax(Eq(14)));
Åsa Persson30ab0152019-08-27 12:22:33 +02004375 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4376
4377 // Trigger adapt down, expect scaled down resolution (480x270@14fps).
4378 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004379 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004380 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionLt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004381 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4382
4383 // Trigger adapt down, expect scaled down framerate (480x270@10fps).
4384 video_stream_encoder_->TriggerQualityLow();
Åsa Persson45b176f2019-09-30 11:19:05 +02004385 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004386 EXPECT_THAT(source_.sink_wants(), FpsLtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004387 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4388
4389 // Trigger adapt up, expect no upscale (target bitrate < min bitrate).
4390 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004391 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004392 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4393
4394 // Trigger adapt up, expect upscaled fps (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004395 OnBitrateUpdated(kMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004396 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004397 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004398 EXPECT_THAT(source_.sink_wants(), FpsGtResolutionEq(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004399 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4400
4401 // Trigger adapt up, expect no upscale in res (target bitrate < min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004402 OnBitrateUpdated(kTooLowMinResolutionBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004403 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004404 InsertFrameAndWaitForEncoded();
Åsa Persson30ab0152019-08-27 12:22:33 +02004405 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4406
4407 // Trigger adapt up, expect upscaled res (target bitrate == min bitrate).
Asa Persson606d3cb2021-10-04 10:07:11 +02004408 OnBitrateUpdated(kResolutionMinBitrate);
Åsa Persson30ab0152019-08-27 12:22:33 +02004409 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson45b176f2019-09-30 11:19:05 +02004410 InsertFrameAndWaitForEncoded();
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004411 EXPECT_THAT(source_.sink_wants(), FpsEqResolutionGt(source_.last_wants()));
Åsa Persson30ab0152019-08-27 12:22:33 +02004412 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4413
Åsa Persson1b247f12019-08-14 17:26:39 +02004414 video_stream_encoder_->Stop();
4415}
4416
mflodmancc3d4422017-08-03 08:27:51 -07004417TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07004418 AdaptsResolutionOnOveruseAndLowQuality_MaintainFramerateMode) {
4419 const int kWidth = 1280;
4420 const int kHeight = 720;
Henrik Boström381d1092020-05-12 18:49:07 +02004421 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004422 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07004423
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004424 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004425 AdaptingFrameForwarder source(&time_controller_);
asaperssond0de2952017-04-21 01:47:31 -07004426 source.set_adaptation_enabled(true);
mflodmancc3d4422017-08-03 08:27:51 -07004427 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004428 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
asaperssond0de2952017-04-21 01:47:31 -07004429
Åsa Persson8c1bf952018-09-13 10:42:19 +02004430 int64_t timestamp_ms = kFrameIntervalMs;
4431 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004432 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004433 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004434 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4435 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4436 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4437 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4438
4439 // Trigger cpu adapt down, expect scaled down resolution (960x540).
mflodmancc3d4422017-08-03 08:27:51 -07004440 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004441 timestamp_ms += kFrameIntervalMs;
4442 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4443 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004444 EXPECT_THAT(source.sink_wants(),
4445 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssond0de2952017-04-21 01:47:31 -07004446 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4447 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4448 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4449 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4450
4451 // Trigger cpu adapt down, expect scaled down resolution (640x360).
mflodmancc3d4422017-08-03 08:27:51 -07004452 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004453 timestamp_ms += kFrameIntervalMs;
4454 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4455 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004456 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004457 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4458 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
4459 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4460 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4461
Jonathan Yubc771b72017-12-08 17:04:29 -08004462 // Trigger cpu adapt down, expect scaled down resolution (480x270).
mflodmancc3d4422017-08-03 08:27:51 -07004463 video_stream_encoder_->TriggerCpuOveruse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004464 timestamp_ms += kFrameIntervalMs;
4465 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4466 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004467 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004468 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4469 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004470 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004471 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4472
Jonathan Yubc771b72017-12-08 17:04:29 -08004473 // Trigger quality adapt down, expect scaled down resolution (320x180).
mflodmancc3d4422017-08-03 08:27:51 -07004474 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004475 timestamp_ms += kFrameIntervalMs;
4476 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4477 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004478 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004479 rtc::VideoSinkWants last_wants = source.sink_wants();
asaperssond0de2952017-04-21 01:47:31 -07004480 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4481 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4482 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4483 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4484
Jonathan Yubc771b72017-12-08 17:04:29 -08004485 // Trigger quality adapt down, expect no change (min resolution reached).
4486 video_stream_encoder_->TriggerQualityLow();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004487 timestamp_ms += kFrameIntervalMs;
4488 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4489 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004490 EXPECT_THAT(source.sink_wants(), FpsMax());
4491 EXPECT_EQ(source.sink_wants().max_pixel_count, last_wants.max_pixel_count);
Jonathan Yubc771b72017-12-08 17:04:29 -08004492 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4493 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4494 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4495 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4496
Evan Shrubsole64469032020-06-11 10:45:29 +02004497 // Trigger quality adapt up, expect upscaled resolution (480x270).
4498 video_stream_encoder_->TriggerQualityHigh();
4499 timestamp_ms += kFrameIntervalMs;
4500 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4501 WaitForEncodedFrame(timestamp_ms);
4502 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
4503 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4504 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4505 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4506 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
4507
4508 // Trigger quality and cpu adapt up since both are most limited, expect
4509 // upscaled resolution (640x360).
Henrik Boström91aa7322020-04-28 12:24:33 +02004510 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004511 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004512 timestamp_ms += kFrameIntervalMs;
4513 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4514 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004515 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Jonathan Yubc771b72017-12-08 17:04:29 -08004516 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
4517 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
4518 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004519 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
Jonathan Yubc771b72017-12-08 17:04:29 -08004520
Evan Shrubsole64469032020-06-11 10:45:29 +02004521 // Trigger quality and cpu adapt up since both are most limited, expect
4522 // upscaled resolution (960x540).
Henrik Boström91aa7322020-04-28 12:24:33 +02004523 video_stream_encoder_->TriggerCpuUnderuse();
Evan Shrubsole64469032020-06-11 10:45:29 +02004524 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004525 timestamp_ms += kFrameIntervalMs;
4526 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4527 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004528 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssond0de2952017-04-21 01:47:31 -07004529 last_wants = source.sink_wants();
Evan Shrubsole64469032020-06-11 10:45:29 +02004530 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
asaperssond0de2952017-04-21 01:47:31 -07004531 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02004532 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
4533 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004534
Evan Shrubsole64469032020-06-11 10:45:29 +02004535 // Trigger cpu adapt up, expect no change since not most limited (960x540).
4536 // However the stats will change since the CPU resource is no longer limited.
Henrik Boström91aa7322020-04-28 12:24:33 +02004537 video_stream_encoder_->TriggerCpuUnderuse();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004538 timestamp_ms += kFrameIntervalMs;
4539 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
4540 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004541 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssond0de2952017-04-21 01:47:31 -07004542 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4543 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004544 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004545 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssond0de2952017-04-21 01:47:31 -07004546
4547 // Trigger quality adapt up, expect no restriction (1280x720).
mflodmancc3d4422017-08-03 08:27:51 -07004548 video_stream_encoder_->TriggerQualityHigh();
Åsa Persson8c1bf952018-09-13 10:42:19 +02004549 timestamp_ms += kFrameIntervalMs;
4550 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004551 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02004552 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02004553 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07004554 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
4555 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
Jonathan Yubc771b72017-12-08 17:04:29 -08004556 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02004557 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
kthelgason5e13d412016-12-01 03:59:51 -08004558
mflodmancc3d4422017-08-03 08:27:51 -07004559 video_stream_encoder_->Stop();
kthelgason5e13d412016-12-01 03:59:51 -08004560}
4561
mflodmancc3d4422017-08-03 08:27:51 -07004562TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
asaperssonfab67072017-04-04 05:51:49 -07004563 const int kWidth = 640;
4564 const int kHeight = 360;
perkj803d97f2016-11-01 11:45:46 -07004565
Henrik Boström381d1092020-05-12 18:49:07 +02004566 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004567 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07004568
perkj803d97f2016-11-01 11:45:46 -07004569 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004570 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004571 WaitForEncodedFrame(i);
perkj803d97f2016-11-01 11:45:46 -07004572 }
4573
mflodmancc3d4422017-08-03 08:27:51 -07004574 video_stream_encoder_->TriggerCpuOveruse();
perkj803d97f2016-11-01 11:45:46 -07004575 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07004576 video_source_.IncomingCapturedFrame(CreateFrame(
4577 SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004578 WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i);
perkj803d97f2016-11-01 11:45:46 -07004579 }
4580
mflodmancc3d4422017-08-03 08:27:51 -07004581 video_stream_encoder_->Stop();
4582 video_stream_encoder_.reset();
perkj803d97f2016-11-01 11:45:46 -07004583 stats_proxy_.reset();
sprangf8ee65e2017-02-28 08:49:33 -08004584
Ying Wangef3998f2019-12-09 13:06:53 +01004585 EXPECT_METRIC_EQ(
4586 1, metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4587 EXPECT_METRIC_EQ(
perkj803d97f2016-11-01 11:45:46 -07004588 1, metrics::NumEvents("WebRTC.Video.CpuLimitedResolutionInPercent", 50));
4589}
4590
mflodmancc3d4422017-08-03 08:27:51 -07004591TEST_F(VideoStreamEncoderTest,
4592 CpuLimitedHistogramIsNotReportedForDisabledDegradation) {
Henrik Boström381d1092020-05-12 18:49:07 +02004593 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004594 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf4e44af2017-04-19 02:01:06 -07004595 const int kWidth = 640;
4596 const int kHeight = 360;
4597
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07004598 video_stream_encoder_->SetSource(&video_source_,
4599 webrtc::DegradationPreference::DISABLED);
asaperssonf4e44af2017-04-19 02:01:06 -07004600
4601 for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
4602 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07004603 WaitForEncodedFrame(i);
asaperssonf4e44af2017-04-19 02:01:06 -07004604 }
4605
mflodmancc3d4422017-08-03 08:27:51 -07004606 video_stream_encoder_->Stop();
4607 video_stream_encoder_.reset();
asaperssonf4e44af2017-04-19 02:01:06 -07004608 stats_proxy_.reset();
4609
4610 EXPECT_EQ(0,
4611 metrics::NumSamples("WebRTC.Video.CpuLimitedResolutionInPercent"));
4612}
4613
Per Kjellanderdcef6412020-10-07 15:09:05 +02004614TEST_F(VideoStreamEncoderTest, ReportsVideoBitrateAllocation) {
4615 ResetEncoder("FAKE", 1, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004616 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02004617 kVideoBitrateAllocation);
sprang57c2fff2017-01-16 06:24:02 -08004618
4619 const int kDefaultFps = 30;
Erik Språng566124a2018-04-23 12:32:22 +02004620 const VideoBitrateAllocation expected_bitrate =
Asa Persson606d3cb2021-10-04 10:07:11 +02004621 SimulcastRateAllocator(fake_encoder_.config())
4622 .Allocate(VideoBitrateAllocationParameters(kLowTargetBitrate.bps(),
Florent Castelli8bbdb5b2019-08-02 15:16:28 +02004623 kDefaultFps));
sprang57c2fff2017-01-16 06:24:02 -08004624
Henrik Boström381d1092020-05-12 18:49:07 +02004625 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004626 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
sprang57c2fff2017-01-16 06:24:02 -08004627
sprang57c2fff2017-01-16 06:24:02 -08004628 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004629 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4630 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004631 EXPECT_EQ(sink_.GetLastVideoBitrateAllocation(), expected_bitrate);
4632 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
4633
Erik Språngd7329ca2019-02-21 21:19:53 +01004634 // Check that encoder has been updated too, not just allocation observer.
Erik Språng9d69cbe2020-10-22 17:44:42 +02004635 EXPECT_TRUE(fake_encoder_.GetAndResetLastRateControlSettings().has_value());
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004636 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004637
Per Kjellanderdcef6412020-10-07 15:09:05 +02004638 // VideoBitrateAllocation not updated on second frame.
sprang57c2fff2017-01-16 06:24:02 -08004639 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004640 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4641 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellanderdcef6412020-10-07 15:09:05 +02004642 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004643 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
sprang57c2fff2017-01-16 06:24:02 -08004644
Per Kjellanderdcef6412020-10-07 15:09:05 +02004645 // VideoBitrateAllocation updated after a process interval.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004646 const int64_t start_time_ms = CurrentTimeMs();
Per Kjellanderd0a8f512020-10-07 11:28:41 +02004647 while (CurrentTimeMs() - start_time_ms < 5 * kProcessIntervalMs) {
Erik Språngd7329ca2019-02-21 21:19:53 +01004648 video_source_.IncomingCapturedFrame(
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02004649 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4650 WaitForEncodedFrame(CurrentTimeMs());
4651 AdvanceTime(TimeDelta::Millis(1) / kDefaultFps);
Erik Språngd7329ca2019-02-21 21:19:53 +01004652 }
Per Kjellanderdcef6412020-10-07 15:09:05 +02004653 EXPECT_GT(sink_.number_of_bitrate_allocations(), 3);
Erik Språngd7329ca2019-02-21 21:19:53 +01004654
mflodmancc3d4422017-08-03 08:27:51 -07004655 video_stream_encoder_->Stop();
sprang57c2fff2017-01-16 06:24:02 -08004656}
4657
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004658TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForVP8Simulcast) {
Per Kjellandera9434842020-10-15 17:53:22 +02004659 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004660 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02004661 kVideoLayersAllocation);
4662
4663 const int kDefaultFps = 30;
4664
4665 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004666 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02004667
4668 video_source_.IncomingCapturedFrame(
4669 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4670 WaitForEncodedFrame(CurrentTimeMs());
4671 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4672 VideoLayersAllocation last_layer_allocation =
4673 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02004674 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02004675 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
4676
4677 VideoBitrateAllocation bitrate_allocation =
Erik Språng9d69cbe2020-10-22 17:44:42 +02004678 fake_encoder_.GetAndResetLastRateControlSettings()->target_bitrate;
Per Kjellandera9434842020-10-15 17:53:22 +02004679 // Check that encoder has been updated too, not just allocation observer.
Asa Persson606d3cb2021-10-04 10:07:11 +02004680 EXPECT_EQ(bitrate_allocation.get_sum_bps(), kLowTargetBitrate.bps());
Per Kjellandera9434842020-10-15 17:53:22 +02004681 AdvanceTime(TimeDelta::Seconds(1) / kDefaultFps);
4682
Erik Språng9d69cbe2020-10-22 17:44:42 +02004683 // VideoLayersAllocation might be updated if frame rate changes.
Per Kjellandera9434842020-10-15 17:53:22 +02004684 int number_of_layers_allocation = 1;
4685 const int64_t start_time_ms = CurrentTimeMs();
4686 while (CurrentTimeMs() - start_time_ms < 10 * kProcessIntervalMs) {
4687 video_source_.IncomingCapturedFrame(
4688 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
4689 WaitForEncodedFrame(CurrentTimeMs());
Per Kjellandera9434842020-10-15 17:53:22 +02004690 if (number_of_layers_allocation != sink_.number_of_layers_allocations()) {
4691 number_of_layers_allocation = sink_.number_of_layers_allocations();
4692 VideoLayersAllocation new_allocation =
4693 sink_.GetLastVideoLayersAllocation();
4694 ASSERT_EQ(new_allocation.active_spatial_layers.size(), 1u);
4695 EXPECT_NE(new_allocation.active_spatial_layers[0].frame_rate_fps,
4696 last_layer_allocation.active_spatial_layers[0].frame_rate_fps);
4697 EXPECT_EQ(new_allocation.active_spatial_layers[0]
4698 .target_bitrate_per_temporal_layer,
4699 last_layer_allocation.active_spatial_layers[0]
4700 .target_bitrate_per_temporal_layer);
4701 last_layer_allocation = new_allocation;
4702 }
4703 }
4704 EXPECT_LE(sink_.number_of_layers_allocations(), 3);
4705 video_stream_encoder_->Stop();
4706}
4707
4708TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004709 ReportsVideoLayersAllocationForVP8WithMiddleLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004710 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4711 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4712 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004713 VideoEncoderConfig video_encoder_config;
4714 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4715 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004716 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004717 video_encoder_config.content_type =
4718 VideoEncoderConfig::ContentType::kRealtimeVideo;
4719 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004720 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004721 VideoEncoder::GetDefaultVp8Settings());
4722 for (auto& layer : video_encoder_config.simulcast_layers) {
4723 layer.num_temporal_layers = 2;
4724 }
4725 // Simulcast layers are used for enabling/disabling streams.
4726 video_encoder_config.simulcast_layers[0].active = true;
4727 video_encoder_config.simulcast_layers[1].active = false;
4728 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004729 ConfigureEncoder(std::move(video_encoder_config),
4730 VideoStreamEncoder::BitrateAllocationCallbackType::
4731 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004732
4733 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004734 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004735
4736 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4737 WaitForEncodedFrame(CurrentTimeMs());
4738 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4739 VideoLayersAllocation last_layer_allocation =
4740 sink_.GetLastVideoLayersAllocation();
4741
4742 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4743 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4744 .target_bitrate_per_temporal_layer,
4745 SizeIs(2));
4746 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4747 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4748 video_stream_encoder_->Stop();
4749}
4750
4751TEST_F(VideoStreamEncoderTest,
Åsa Perssona7e34d32021-01-20 15:36:13 +01004752 ReportsVideoLayersAllocationForVP8WithMiddleAndHighestLayerDisabled) {
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004753 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4754 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4755 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004756 VideoEncoderConfig video_encoder_config;
4757 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP8,
4758 /* num_streams*/ 3, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004759 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004760 video_encoder_config.content_type =
4761 VideoEncoderConfig::ContentType::kRealtimeVideo;
4762 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004763 rtc::make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004764 VideoEncoder::GetDefaultVp8Settings());
4765 for (auto& layer : video_encoder_config.simulcast_layers) {
4766 layer.num_temporal_layers = 2;
4767 }
4768 // Simulcast layers are used for enabling/disabling streams.
4769 video_encoder_config.simulcast_layers[0].active = true;
4770 video_encoder_config.simulcast_layers[1].active = false;
4771 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004772 ConfigureEncoder(std::move(video_encoder_config),
4773 VideoStreamEncoder::BitrateAllocationCallbackType::
4774 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004775
4776 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004777 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004778
4779 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4780 WaitForEncodedFrame(CurrentTimeMs());
4781 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4782 VideoLayersAllocation last_layer_allocation =
4783 sink_.GetLastVideoLayersAllocation();
4784
4785 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
4786 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4787 .target_bitrate_per_temporal_layer,
4788 SizeIs(2));
4789 EXPECT_LT(last_layer_allocation.active_spatial_layers[0].width, 1280);
4790
4791 video_stream_encoder_->Stop();
4792}
4793
4794TEST_F(VideoStreamEncoderTest,
4795 ReportsVideoLayersAllocationForV9SvcWithTemporalLayerSupport) {
4796 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4797 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004798 VideoEncoderConfig video_encoder_config;
4799 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4800 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004801 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004802 video_encoder_config.content_type =
4803 VideoEncoderConfig::ContentType::kRealtimeVideo;
4804 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4805 vp9_settings.numberOfSpatialLayers = 2;
4806 vp9_settings.numberOfTemporalLayers = 2;
4807 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4808 vp9_settings.automaticResizeOn = false;
4809 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004810 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004811 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004812 ConfigureEncoder(std::move(video_encoder_config),
4813 VideoStreamEncoder::BitrateAllocationCallbackType::
4814 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004815
4816 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004817 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004818
4819 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4820 WaitForEncodedFrame(CurrentTimeMs());
4821 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4822 VideoLayersAllocation last_layer_allocation =
4823 sink_.GetLastVideoLayersAllocation();
4824
4825 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4826 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4827 .target_bitrate_per_temporal_layer,
4828 SizeIs(2));
4829 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4830 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 360);
4831 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
4832 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4833 .target_bitrate_per_temporal_layer,
4834 SizeIs(2));
4835 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4836 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].height, 720);
4837 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].frame_rate_fps, 30);
4838
4839 // Since full SVC is used, expect the top layer to utilize the full target
4840 // rate.
4841 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4842 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004843 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004844 video_stream_encoder_->Stop();
4845}
4846
4847TEST_F(VideoStreamEncoderTest,
4848 ReportsVideoLayersAllocationForV9SvcWithoutTemporalLayerSupport) {
4849 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, false);
4850 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, false);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004851 VideoEncoderConfig video_encoder_config;
4852 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4853 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004854 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004855 video_encoder_config.content_type =
4856 VideoEncoderConfig::ContentType::kRealtimeVideo;
4857 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4858 vp9_settings.numberOfSpatialLayers = 2;
4859 vp9_settings.numberOfTemporalLayers = 2;
4860 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4861 vp9_settings.automaticResizeOn = false;
4862 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004863 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004864 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004865 ConfigureEncoder(std::move(video_encoder_config),
4866 VideoStreamEncoder::BitrateAllocationCallbackType::
4867 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004868
4869 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004870 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004871
4872 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4873 WaitForEncodedFrame(CurrentTimeMs());
4874 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4875 VideoLayersAllocation last_layer_allocation =
4876 sink_.GetLastVideoLayersAllocation();
4877
4878 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4879 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4880 .target_bitrate_per_temporal_layer,
4881 SizeIs(1));
4882 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4883 .target_bitrate_per_temporal_layer,
4884 SizeIs(1));
4885 // Since full SVC is used, expect the top layer to utilize the full target
4886 // rate.
4887 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4888 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02004889 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004890 video_stream_encoder_->Stop();
4891}
4892
4893TEST_F(VideoStreamEncoderTest,
4894 ReportsVideoLayersAllocationForVP9KSvcWithTemporalLayerSupport) {
4895 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4896 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004897 VideoEncoderConfig video_encoder_config;
4898 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4899 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004900 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004901 video_encoder_config.content_type =
4902 VideoEncoderConfig::ContentType::kRealtimeVideo;
4903 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4904 vp9_settings.numberOfSpatialLayers = 2;
4905 vp9_settings.numberOfTemporalLayers = 2;
4906 vp9_settings.interLayerPred = InterLayerPredMode::kOnKeyPic;
4907 vp9_settings.automaticResizeOn = false;
4908 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004909 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004910 vp9_settings);
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004911 ConfigureEncoder(std::move(video_encoder_config),
4912 VideoStreamEncoder::BitrateAllocationCallbackType::
4913 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004914
4915 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004916 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004917
4918 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4919 WaitForEncodedFrame(CurrentTimeMs());
4920 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4921 VideoLayersAllocation last_layer_allocation =
4922 sink_.GetLastVideoLayersAllocation();
4923
4924 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4925 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4926 .target_bitrate_per_temporal_layer,
4927 SizeIs(2));
4928 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4929 .target_bitrate_per_temporal_layer,
4930 SizeIs(2));
4931 // Since KSVC is, spatial layers are independend except on key frames.
4932 EXPECT_LT(last_layer_allocation.active_spatial_layers[1]
4933 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004934 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004935 video_stream_encoder_->Stop();
4936}
4937
4938TEST_F(VideoStreamEncoderTest,
4939 ReportsVideoLayersAllocationForV9SvcWithLowestLayerDisabled) {
4940 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4941 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4942 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004943 VideoEncoderConfig video_encoder_config;
4944 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
4945 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02004946 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004947 video_encoder_config.content_type =
4948 VideoEncoderConfig::ContentType::kRealtimeVideo;
4949 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
4950 vp9_settings.numberOfSpatialLayers = 3;
4951 vp9_settings.numberOfTemporalLayers = 2;
4952 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
4953 vp9_settings.automaticResizeOn = false;
4954 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02004955 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004956 vp9_settings);
4957 // Simulcast layers are used for enabling/disabling streams.
4958 video_encoder_config.simulcast_layers.resize(3);
4959 video_encoder_config.simulcast_layers[0].active = false;
4960 video_encoder_config.simulcast_layers[1].active = true;
4961 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01004962 ConfigureEncoder(std::move(video_encoder_config),
4963 VideoStreamEncoder::BitrateAllocationCallbackType::
4964 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004965
4966 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02004967 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004968
4969 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
4970 WaitForEncodedFrame(CurrentTimeMs());
4971 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
4972 VideoLayersAllocation last_layer_allocation =
4973 sink_.GetLastVideoLayersAllocation();
4974
4975 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
4976 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
4977 .target_bitrate_per_temporal_layer,
4978 SizeIs(2));
4979 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 640);
4980 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
4981
4982 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 1280);
4983 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
4984 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
4985 .target_bitrate_per_temporal_layer,
4986 SizeIs(2));
4987 // Since full SVC is used, expect the top layer to utilize the full target
4988 // rate.
4989 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1]
4990 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02004991 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01004992 video_stream_encoder_->Stop();
4993}
4994
4995TEST_F(VideoStreamEncoderTest,
4996 ReportsVideoLayersAllocationForV9SvcWithHighestLayerDisabled) {
4997 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
4998 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
4999 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005000 VideoEncoderConfig video_encoder_config;
5001 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5002 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005003 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005004 video_encoder_config.content_type =
5005 VideoEncoderConfig::ContentType::kRealtimeVideo;
5006 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5007 vp9_settings.numberOfSpatialLayers = 3;
5008 vp9_settings.numberOfTemporalLayers = 2;
5009 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5010 vp9_settings.automaticResizeOn = false;
5011 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005012 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005013 vp9_settings);
5014 // Simulcast layers are used for enabling/disabling streams.
5015 video_encoder_config.simulcast_layers.resize(3);
5016 video_encoder_config.simulcast_layers[2].active = false;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005017 ConfigureEncoder(std::move(video_encoder_config),
5018 VideoStreamEncoder::BitrateAllocationCallbackType::
5019 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005020
5021 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005022 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005023
5024 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5025 WaitForEncodedFrame(CurrentTimeMs());
5026 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5027 VideoLayersAllocation last_layer_allocation =
5028 sink_.GetLastVideoLayersAllocation();
5029
5030 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(2));
5031 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5032 .target_bitrate_per_temporal_layer,
5033 SizeIs(2));
5034 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 320);
5035 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5036
5037 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].width, 640);
5038 EXPECT_EQ(last_layer_allocation.active_spatial_layers[1].spatial_id, 1);
5039 EXPECT_THAT(last_layer_allocation.active_spatial_layers[1]
5040 .target_bitrate_per_temporal_layer,
5041 SizeIs(2));
5042 video_stream_encoder_->Stop();
5043}
5044
5045TEST_F(VideoStreamEncoderTest,
5046 ReportsVideoLayersAllocationForV9SvcWithAllButHighestLayerDisabled) {
5047 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx=*/0, true);
5048 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 1, true);
5049 fake_encoder_.SetTemporalLayersSupported(/*spatial_idx*/ 2, true);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005050 VideoEncoderConfig video_encoder_config;
5051 test::FillEncoderConfiguration(VideoCodecType::kVideoCodecVP9,
5052 /* num_streams*/ 1, &video_encoder_config);
Asa Persson606d3cb2021-10-04 10:07:11 +02005053 video_encoder_config.max_bitrate_bps = 2 * kTargetBitrate.bps();
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005054 video_encoder_config.content_type =
5055 VideoEncoderConfig::ContentType::kRealtimeVideo;
5056 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5057 vp9_settings.numberOfSpatialLayers = 3;
5058 vp9_settings.numberOfTemporalLayers = 2;
5059 vp9_settings.interLayerPred = InterLayerPredMode::kOn;
5060 vp9_settings.automaticResizeOn = false;
5061 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005062 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005063 vp9_settings);
5064 // Simulcast layers are used for enabling/disabling streams.
5065 video_encoder_config.simulcast_layers.resize(3);
5066 video_encoder_config.simulcast_layers[0].active = false;
5067 video_encoder_config.simulcast_layers[1].active = false;
5068 video_encoder_config.simulcast_layers[2].active = true;
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005069 ConfigureEncoder(std::move(video_encoder_config),
5070 VideoStreamEncoder::BitrateAllocationCallbackType::
5071 kVideoLayersAllocation);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005072
5073 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005074 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005075
5076 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5077 WaitForEncodedFrame(CurrentTimeMs());
5078 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5079 VideoLayersAllocation last_layer_allocation =
5080 sink_.GetLastVideoLayersAllocation();
5081
5082 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5083 EXPECT_THAT(last_layer_allocation.active_spatial_layers[0]
5084 .target_bitrate_per_temporal_layer,
5085 SizeIs(2));
5086 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5087 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].spatial_id, 0);
5088 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5089 .target_bitrate_per_temporal_layer[1],
Asa Persson606d3cb2021-10-04 10:07:11 +02005090 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005091 video_stream_encoder_->Stop();
5092}
5093
5094TEST_F(VideoStreamEncoderTest, ReportsVideoLayersAllocationForH264) {
5095 ResetEncoder("H264", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005096 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005097 kVideoLayersAllocation);
5098 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005099 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005100
5101 video_source_.IncomingCapturedFrame(CreateFrame(CurrentTimeMs(), 1280, 720));
5102 WaitForEncodedFrame(CurrentTimeMs());
5103 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5104 VideoLayersAllocation last_layer_allocation =
5105 sink_.GetLastVideoLayersAllocation();
5106
5107 ASSERT_THAT(last_layer_allocation.active_spatial_layers, SizeIs(1));
5108 ASSERT_THAT(last_layer_allocation.active_spatial_layers[0]
5109 .target_bitrate_per_temporal_layer,
5110 SizeIs(1));
5111 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5112 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005113 kTargetBitrate);
Per Kjellanderf86cf4c2020-12-30 15:27:35 +01005114 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].width, 1280);
5115 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].height, 720);
5116 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0].frame_rate_fps, 30);
5117 video_stream_encoder_->Stop();
5118}
5119
5120TEST_F(VideoStreamEncoderTest,
Per Kjellandera9434842020-10-15 17:53:22 +02005121 ReportsUpdatedVideoLayersAllocationWhenBweChanges) {
5122 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005123 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellandera9434842020-10-15 17:53:22 +02005124 kVideoLayersAllocation);
5125
5126 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005127 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005128
5129 video_source_.IncomingCapturedFrame(
5130 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5131 WaitForEncodedFrame(CurrentTimeMs());
5132 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5133 VideoLayersAllocation last_layer_allocation =
5134 sink_.GetLastVideoLayersAllocation();
Asa Persson606d3cb2021-10-04 10:07:11 +02005135 // kLowTargetBitrate is only enough for one spatial layer.
Per Kjellandera9434842020-10-15 17:53:22 +02005136 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 1u);
5137 EXPECT_EQ(last_layer_allocation.active_spatial_layers[0]
5138 .target_bitrate_per_temporal_layer[0],
Asa Persson606d3cb2021-10-04 10:07:11 +02005139 kLowTargetBitrate);
Per Kjellandera9434842020-10-15 17:53:22 +02005140
5141 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005142 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5143 0, 0, 0);
Per Kjellandera9434842020-10-15 17:53:22 +02005144 video_source_.IncomingCapturedFrame(
5145 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5146 WaitForEncodedFrame(CurrentTimeMs());
5147
5148 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5149 last_layer_allocation = sink_.GetLastVideoLayersAllocation();
5150 ASSERT_EQ(last_layer_allocation.active_spatial_layers.size(), 2u);
5151 EXPECT_GT(last_layer_allocation.active_spatial_layers[1]
5152 .target_bitrate_per_temporal_layer[0],
5153 DataRate::Zero());
5154
5155 video_stream_encoder_->Stop();
5156}
5157
Per Kjellander4190ce92020-12-15 17:24:55 +01005158TEST_F(VideoStreamEncoderTest,
5159 ReportsUpdatedVideoLayersAllocationWhenResolutionChanges) {
5160 ResetEncoder("VP8", /*num_streams*/ 2, 1, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005161 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellander4190ce92020-12-15 17:24:55 +01005162 kVideoLayersAllocation);
5163
5164 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005165 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
5166 0, 0, 0);
Per Kjellander4190ce92020-12-15 17:24:55 +01005167
5168 video_source_.IncomingCapturedFrame(
5169 CreateFrame(CurrentTimeMs(), codec_width_, codec_height_));
5170 WaitForEncodedFrame(CurrentTimeMs());
5171 EXPECT_EQ(sink_.number_of_layers_allocations(), 1);
5172 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5173 SizeIs(2));
5174 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5175 codec_width_);
5176 EXPECT_EQ(
5177 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5178 codec_height_);
5179
5180 video_source_.IncomingCapturedFrame(
5181 CreateFrame(CurrentTimeMs(), codec_width_ / 2, codec_height_ / 2));
5182 WaitForEncodedFrame(CurrentTimeMs());
5183 EXPECT_EQ(sink_.number_of_layers_allocations(), 2);
5184 ASSERT_THAT(sink_.GetLastVideoLayersAllocation().active_spatial_layers,
5185 SizeIs(2));
5186 EXPECT_EQ(sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].width,
5187 codec_width_ / 2);
5188 EXPECT_EQ(
5189 sink_.GetLastVideoLayersAllocation().active_spatial_layers[1].height,
5190 codec_height_ / 2);
5191
5192 video_stream_encoder_->Stop();
5193}
5194
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005195TEST_F(VideoStreamEncoderTest, TemporalLayersNotDisabledIfSupported) {
5196 // 2 TLs configured, temporal layers supported by encoder.
5197 const int kNumTemporalLayers = 2;
Per Kjellanderdcef6412020-10-07 15:09:05 +02005198 ResetEncoder("VP8", 1, kNumTemporalLayers, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005199 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005200 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005201 fake_encoder_.SetTemporalLayersSupported(0, true);
5202
5203 // Bitrate allocated across temporal layers.
Asa Persson606d3cb2021-10-04 10:07:11 +02005204 const int kTl0Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005205 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005206 kNumTemporalLayers, /*temporal_id*/ 0,
5207 /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005208 const int kTl1Bps = kTargetBitrate.bps() *
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005209 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005210 kNumTemporalLayers, /*temporal_id*/ 1,
5211 /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005212 VideoBitrateAllocation expected_bitrate;
5213 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTl0Bps);
5214 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kTl1Bps - kTl0Bps);
5215
5216 VerifyAllocatedBitrate(expected_bitrate);
5217 video_stream_encoder_->Stop();
5218}
5219
5220TEST_F(VideoStreamEncoderTest, TemporalLayersDisabledIfNotSupported) {
5221 // 2 TLs configured, temporal layers not supported by encoder.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005222 ResetEncoder("VP8", 1, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005223 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005224 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005225 fake_encoder_.SetTemporalLayersSupported(0, false);
5226
5227 // Temporal layers not supported by the encoder.
5228 // Total bitrate should be at ti:0.
5229 VideoBitrateAllocation expected_bitrate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005230 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kTargetBitrate.bps());
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005231
5232 VerifyAllocatedBitrate(expected_bitrate);
5233 video_stream_encoder_->Stop();
5234}
5235
5236TEST_F(VideoStreamEncoderTest, VerifyBitrateAllocationForTwoStreams) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005237 webrtc::test::ScopedKeyValueConfig field_trials(
5238 field_trials_,
Per Kjellanderdcef6412020-10-07 15:09:05 +02005239 "WebRTC-Video-QualityScalerSettings/"
5240 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5241 // Reset encoder for field trials to take effect.
5242 ConfigureEncoder(video_encoder_config_.Copy());
5243
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005244 // 2 TLs configured, temporal layers only supported for first stream.
Per Kjellanderdcef6412020-10-07 15:09:05 +02005245 ResetEncoder("VP8", 2, /*num_temporal_layers*/ 2, 1, /*screenshare*/ false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01005246 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02005247 kVideoBitrateAllocation);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005248 fake_encoder_.SetTemporalLayersSupported(0, true);
5249 fake_encoder_.SetTemporalLayersSupported(1, false);
5250
5251 const int kS0Bps = 150000;
5252 const int kS0Tl0Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005253 kS0Bps *
5254 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5255 /*num_layers*/ 2, /*temporal_id*/ 0, /*base_heavy_tl3_alloc*/ false);
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005256 const int kS0Tl1Bps =
Rasmus Brandt2b9317a2019-10-30 13:01:46 +01005257 kS0Bps *
5258 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
5259 /*num_layers*/ 2, /*temporal_id*/ 1, /*base_heavy_tl3_alloc*/ false);
Asa Persson606d3cb2021-10-04 10:07:11 +02005260 const int kS1Bps = kTargetBitrate.bps() - kS0Tl1Bps;
Åsa Perssonc29cb2c2019-03-25 12:06:59 +01005261 // Temporal layers not supported by si:1.
5262 VideoBitrateAllocation expected_bitrate;
5263 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 0, kS0Tl0Bps);
5264 expected_bitrate.SetBitrate(/*si*/ 0, /*ti*/ 1, kS0Tl1Bps - kS0Tl0Bps);
5265 expected_bitrate.SetBitrate(/*si*/ 1, /*ti*/ 0, kS1Bps);
5266
5267 VerifyAllocatedBitrate(expected_bitrate);
5268 video_stream_encoder_->Stop();
5269}
5270
Niels Möller7dc26b72017-12-06 10:27:48 +01005271TEST_F(VideoStreamEncoderTest, OveruseDetectorUpdatedOnReconfigureAndAdaption) {
5272 const int kFrameWidth = 1280;
5273 const int kFrameHeight = 720;
5274 const int kFramerate = 24;
5275
Henrik Boström381d1092020-05-12 18:49:07 +02005276 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005277 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005278 test::FrameForwarder source;
5279 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005280 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005281
5282 // Insert a single frame, triggering initial configuration.
5283 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5284 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5285
5286 EXPECT_EQ(
5287 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5288 kDefaultFramerate);
5289
5290 // Trigger reconfigure encoder (without resetting the entire instance).
5291 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005292 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5293 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005294 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005295 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005296 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005297 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5298
5299 // Detector should be updated with fps limit from codec config.
5300 EXPECT_EQ(
5301 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5302 kFramerate);
5303
5304 // Trigger overuse, max framerate should be reduced.
5305 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5306 stats.input_frame_rate = kFramerate;
5307 stats_proxy_->SetMockStats(stats);
5308 video_stream_encoder_->TriggerCpuOveruse();
5309 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5310 int adapted_framerate =
5311 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5312 EXPECT_LT(adapted_framerate, kFramerate);
5313
5314 // Trigger underuse, max framerate should go back to codec configured fps.
5315 // Set extra low fps, to make sure it's actually reset, not just incremented.
5316 stats = stats_proxy_->GetStats();
5317 stats.input_frame_rate = adapted_framerate / 2;
5318 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005319 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005320 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5321 EXPECT_EQ(
5322 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5323 kFramerate);
5324
5325 video_stream_encoder_->Stop();
5326}
5327
5328TEST_F(VideoStreamEncoderTest,
5329 OveruseDetectorUpdatedRespectsFramerateAfterUnderuse) {
5330 const int kFrameWidth = 1280;
5331 const int kFrameHeight = 720;
5332 const int kLowFramerate = 15;
5333 const int kHighFramerate = 25;
5334
Henrik Boström381d1092020-05-12 18:49:07 +02005335 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005336 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller7dc26b72017-12-06 10:27:48 +01005337 test::FrameForwarder source;
5338 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005339 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
Niels Möller7dc26b72017-12-06 10:27:48 +01005340
5341 // Trigger initial configuration.
5342 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005343 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5344 video_encoder_config.simulcast_layers[0].max_framerate = kLowFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005345 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
Niels Möller7dc26b72017-12-06 10:27:48 +01005346 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
Åsa Persson17107062020-10-08 08:57:51 +02005347 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
Niels Möllerf1338562018-04-26 09:51:47 +02005348 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005349 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5350
5351 EXPECT_EQ(
5352 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5353 kLowFramerate);
5354
5355 // Trigger overuse, max framerate should be reduced.
5356 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5357 stats.input_frame_rate = kLowFramerate;
5358 stats_proxy_->SetMockStats(stats);
5359 video_stream_encoder_->TriggerCpuOveruse();
5360 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5361 int adapted_framerate =
5362 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
5363 EXPECT_LT(adapted_framerate, kLowFramerate);
5364
5365 // Reconfigure the encoder with a new (higher max framerate), max fps should
5366 // still respect the adaptation.
Åsa Persson17107062020-10-08 08:57:51 +02005367 video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
Niels Möller7dc26b72017-12-06 10:27:48 +01005368 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
5369 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005370 kMaxPayloadLength);
Niels Möller7dc26b72017-12-06 10:27:48 +01005371 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5372
5373 EXPECT_EQ(
5374 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5375 adapted_framerate);
5376
5377 // Trigger underuse, max framerate should go back to codec configured fps.
5378 stats = stats_proxy_->GetStats();
5379 stats.input_frame_rate = adapted_framerate;
5380 stats_proxy_->SetMockStats(stats);
Henrik Boström91aa7322020-04-28 12:24:33 +02005381 video_stream_encoder_->TriggerCpuUnderuse();
Niels Möller7dc26b72017-12-06 10:27:48 +01005382 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5383 EXPECT_EQ(
5384 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5385 kHighFramerate);
5386
5387 video_stream_encoder_->Stop();
5388}
5389
mflodmancc3d4422017-08-03 08:27:51 -07005390TEST_F(VideoStreamEncoderTest,
5391 OveruseDetectorUpdatedOnDegradationPreferenceChange) {
sprangfda496a2017-06-15 04:21:07 -07005392 const int kFrameWidth = 1280;
5393 const int kFrameHeight = 720;
5394 const int kFramerate = 24;
5395
Henrik Boström381d1092020-05-12 18:49:07 +02005396 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005397 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangfda496a2017-06-15 04:21:07 -07005398 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07005399 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005400 &source, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangfda496a2017-06-15 04:21:07 -07005401
5402 // Trigger initial configuration.
5403 VideoEncoderConfig video_encoder_config;
Åsa Persson17107062020-10-08 08:57:51 +02005404 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5405 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02005406 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
sprangfda496a2017-06-15 04:21:07 -07005407 source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight));
mflodmancc3d4422017-08-03 08:27:51 -07005408 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005409 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07005410 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprangfda496a2017-06-15 04:21:07 -07005411
Niels Möller7dc26b72017-12-06 10:27:48 +01005412 EXPECT_EQ(
5413 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5414 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005415
5416 // Trigger overuse, max framerate should be reduced.
5417 VideoSendStream::Stats stats = stats_proxy_->GetStats();
5418 stats.input_frame_rate = kFramerate;
5419 stats_proxy_->SetMockStats(stats);
mflodmancc3d4422017-08-03 08:27:51 -07005420 video_stream_encoder_->TriggerCpuOveruse();
5421 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Niels Möller7dc26b72017-12-06 10:27:48 +01005422 int adapted_framerate =
5423 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate();
sprangfda496a2017-06-15 04:21:07 -07005424 EXPECT_LT(adapted_framerate, kFramerate);
5425
5426 // Change degradation preference to not enable framerate scaling. Target
5427 // framerate should be changed to codec defined limit.
Henrik Boström381d1092020-05-12 18:49:07 +02005428 video_stream_encoder_->SetSourceAndWaitForFramerateUpdated(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005429 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Niels Möller7dc26b72017-12-06 10:27:48 +01005430 EXPECT_EQ(
5431 video_stream_encoder_->overuse_detector_proxy_->GetLastTargetFramerate(),
5432 kFramerate);
sprangfda496a2017-06-15 04:21:07 -07005433
mflodmancc3d4422017-08-03 08:27:51 -07005434 video_stream_encoder_->Stop();
sprangfda496a2017-06-15 04:21:07 -07005435}
5436
mflodmancc3d4422017-08-03 08:27:51 -07005437TEST_F(VideoStreamEncoderTest, DropsFramesAndScalesWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005438 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005439 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005440 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5441 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5442 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005443 const int kWidth = 640;
5444 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005445
asaperssonfab67072017-04-04 05:51:49 -07005446 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005447
5448 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005449 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005450
5451 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005452 EXPECT_TRUE_WAIT(
5453 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005454
sprangc5d62e22017-04-02 23:53:04 -07005455 int last_pixel_count = video_source_.sink_wants().max_pixel_count;
kthelgason2bc68642017-02-07 07:02:22 -08005456
asaperssonfab67072017-04-04 05:51:49 -07005457 // Next frame is scaled.
kthelgason2bc68642017-02-07 07:02:22 -08005458 video_source_.IncomingCapturedFrame(
asaperssonfab67072017-04-04 05:51:49 -07005459 CreateFrame(2, kWidth * 3 / 4, kHeight * 3 / 4));
kthelgason2bc68642017-02-07 07:02:22 -08005460
5461 // Expect to drop this frame, the wait should time out.
sprang4847ae62017-06-27 07:06:52 -07005462 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005463
Henrik Boström2671dac2020-05-19 16:29:09 +02005464 EXPECT_TRUE_WAIT(
5465 video_source_.sink_wants().max_pixel_count < last_pixel_count, 5000);
kthelgason2bc68642017-02-07 07:02:22 -08005466
mflodmancc3d4422017-08-03 08:27:51 -07005467 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005468}
5469
mflodmancc3d4422017-08-03 08:27:51 -07005470TEST_F(VideoStreamEncoderTest,
5471 NumberOfDroppedFramesLimitedWhenBitrateIsTooLow) {
asaperssonfab67072017-04-04 05:51:49 -07005472 const int kTooLowBitrateForFrameSizeBps = 10000;
Henrik Boström381d1092020-05-12 18:49:07 +02005473 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005474 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5475 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5476 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
asaperssonfab67072017-04-04 05:51:49 -07005477 const int kWidth = 640;
5478 const int kHeight = 360;
kthelgason2bc68642017-02-07 07:02:22 -08005479
5480 // We expect the n initial frames to get dropped.
5481 int i;
5482 for (i = 1; i <= kMaxInitialFramedrop; ++i) {
asaperssonfab67072017-04-04 05:51:49 -07005483 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005484 ExpectDroppedFrame();
kthelgason2bc68642017-02-07 07:02:22 -08005485 }
5486 // The n+1th frame should not be dropped, even though it's size is too large.
asaperssonfab67072017-04-04 05:51:49 -07005487 video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07005488 WaitForEncodedFrame(i);
kthelgason2bc68642017-02-07 07:02:22 -08005489
5490 // Expect the sink_wants to specify a scaled frame.
asaperssonfab67072017-04-04 05:51:49 -07005491 EXPECT_LT(video_source_.sink_wants().max_pixel_count, kWidth * kHeight);
kthelgason2bc68642017-02-07 07:02:22 -08005492
mflodmancc3d4422017-08-03 08:27:51 -07005493 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005494}
5495
mflodmancc3d4422017-08-03 08:27:51 -07005496TEST_F(VideoStreamEncoderTest,
5497 InitialFrameDropOffWithMaintainResolutionPreference) {
asaperssonfab67072017-04-04 05:51:49 -07005498 const int kWidth = 640;
5499 const int kHeight = 360;
Henrik Boström381d1092020-05-12 18:49:07 +02005500 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005501 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
kthelgason2bc68642017-02-07 07:02:22 -08005502
5503 // Set degradation preference.
mflodmancc3d4422017-08-03 08:27:51 -07005504 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005505 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
kthelgason2bc68642017-02-07 07:02:22 -08005506
asaperssonfab67072017-04-04 05:51:49 -07005507 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgason2bc68642017-02-07 07:02:22 -08005508 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005509 WaitForEncodedFrame(1);
kthelgason2bc68642017-02-07 07:02:22 -08005510
mflodmancc3d4422017-08-03 08:27:51 -07005511 video_stream_encoder_->Stop();
kthelgason2bc68642017-02-07 07:02:22 -08005512}
5513
mflodmancc3d4422017-08-03 08:27:51 -07005514TEST_F(VideoStreamEncoderTest, InitialFrameDropOffWhenEncoderDisabledScaling) {
asaperssonfab67072017-04-04 05:51:49 -07005515 const int kWidth = 640;
5516 const int kHeight = 360;
kthelgasonad9010c2017-02-14 00:46:51 -08005517 fake_encoder_.SetQualityScaling(false);
Niels Möller4db138e2018-04-19 09:04:13 +02005518
5519 VideoEncoderConfig video_encoder_config;
5520 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
5521 // Make format different, to force recreation of encoder.
5522 video_encoder_config.video_format.parameters["foo"] = "foo";
5523 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02005524 kMaxPayloadLength);
Henrik Boström381d1092020-05-12 18:49:07 +02005525 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005526 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
asapersson09f05612017-05-15 23:40:18 -07005527
kthelgasonb83797b2017-02-14 11:57:25 -08005528 // Force quality scaler reconfiguration by resetting the source.
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07005529 video_stream_encoder_->SetSource(&video_source_,
5530 webrtc::DegradationPreference::BALANCED);
kthelgasonad9010c2017-02-14 00:46:51 -08005531
asaperssonfab67072017-04-04 05:51:49 -07005532 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
kthelgasonad9010c2017-02-14 00:46:51 -08005533 // Frame should not be dropped, even if it's too large.
sprang4847ae62017-06-27 07:06:52 -07005534 WaitForEncodedFrame(1);
kthelgasonad9010c2017-02-14 00:46:51 -08005535
mflodmancc3d4422017-08-03 08:27:51 -07005536 video_stream_encoder_->Stop();
kthelgasonad9010c2017-02-14 00:46:51 -08005537 fake_encoder_.SetQualityScaling(true);
5538}
5539
Åsa Persson139f4dc2019-08-02 09:29:58 +02005540TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenBweDrops) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005541 webrtc::test::ScopedKeyValueConfig field_trials(
5542 field_trials_,
Åsa Persson139f4dc2019-08-02 09:29:58 +02005543 "WebRTC-Video-QualityScalerSettings/"
5544 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5545 // Reset encoder for field trials to take effect.
5546 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005547 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5548 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Åsa Persson139f4dc2019-08-02 09:29:58 +02005549 const int kWidth = 640;
5550 const int kHeight = 360;
5551
Henrik Boström381d1092020-05-12 18:49:07 +02005552 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005553 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005554 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5555 // Frame should not be dropped.
5556 WaitForEncodedFrame(1);
5557
Henrik Boström381d1092020-05-12 18:49:07 +02005558 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005559 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5560 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5561 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005562 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5563 // Frame should not be dropped.
5564 WaitForEncodedFrame(2);
5565
Henrik Boström381d1092020-05-12 18:49:07 +02005566 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01005567 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5568 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5569 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005570 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5571 // Expect to drop this frame, the wait should time out.
5572 ExpectDroppedFrame();
5573
5574 // Expect the sink_wants to specify a scaled frame.
Henrik Boström2671dac2020-05-19 16:29:09 +02005575 EXPECT_TRUE_WAIT(
5576 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
Åsa Persson139f4dc2019-08-02 09:29:58 +02005577 video_stream_encoder_->Stop();
5578}
5579
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005580TEST_F(VideoStreamEncoderTest,
5581 InitialFrameDropNotReactivatedWhenBweDropsWhenScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005582 webrtc::test::ScopedKeyValueConfig field_trials(
5583 field_trials_,
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005584 "WebRTC-Video-QualityScalerSettings/"
5585 "initial_bitrate_interval_ms:1000,initial_bitrate_factor:0.2/");
5586 fake_encoder_.SetQualityScaling(false);
5587 ConfigureEncoder(video_encoder_config_.Copy());
Asa Persson606d3cb2021-10-04 10:07:11 +02005588 const int kNotTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.2;
5589 const int kTooLowBitrateForFrameSizeBps = kTargetBitrate.bps() * 0.19;
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005590 const int kWidth = 640;
5591 const int kHeight = 360;
5592
5593 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005594 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005595 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5596 // Frame should not be dropped.
5597 WaitForEncodedFrame(1);
5598
5599 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5600 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5601 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps),
5602 DataRate::BitsPerSec(kNotTooLowBitrateForFrameSizeBps), 0, 0, 0);
5603 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5604 // Frame should not be dropped.
5605 WaitForEncodedFrame(2);
5606
5607 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
5608 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5609 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps),
5610 DataRate::BitsPerSec(kTooLowBitrateForFrameSizeBps), 0, 0, 0);
5611 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5612 // Not dropped since quality scaling is disabled.
5613 WaitForEncodedFrame(3);
5614
5615 // Expect the sink_wants to specify a scaled frame.
Evan Shrubsole85728412020-08-25 13:12:12 +02005616 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsolee3da1d32020-08-14 15:58:33 +02005617 EXPECT_THAT(video_source_.sink_wants(), ResolutionMax());
5618
5619 video_stream_encoder_->Stop();
5620}
5621
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005622TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005623 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005624 // Set simulcast.
5625 ResetEncoder("VP8", 3, 1, 1, false);
5626 fake_encoder_.SetQualityScaling(true);
5627 const int kWidth = 1280;
5628 const int kHeight = 720;
5629 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005630 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005631 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5632 // Frame should not be dropped.
5633 WaitForEncodedFrame(1);
5634
5635 // Trigger QVGA "singlecast"
5636 // Update the config.
5637 VideoEncoderConfig video_encoder_config;
5638 test::FillEncoderConfiguration(PayloadStringToCodecType("VP8"), 3,
5639 &video_encoder_config);
Åsa Persson7f354f82021-02-04 15:52:15 +01005640 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005641 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Åsa Persson7f354f82021-02-04 15:52:15 +01005642 "VP8", /*max qp*/ 56, /*screencast*/ false,
5643 /*screenshare enabled*/ false);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005644 for (auto& layer : video_encoder_config.simulcast_layers) {
5645 layer.num_temporal_layers = 1;
5646 layer.max_framerate = kDefaultFramerate;
5647 }
Asa Persson606d3cb2021-10-04 10:07:11 +02005648 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005649 video_encoder_config.content_type =
5650 VideoEncoderConfig::ContentType::kRealtimeVideo;
5651
5652 video_encoder_config.simulcast_layers[0].active = true;
5653 video_encoder_config.simulcast_layers[1].active = false;
5654 video_encoder_config.simulcast_layers[2].active = false;
5655
5656 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5657 kMaxPayloadLength);
5658 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5659
5660 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5661 // Frame should not be dropped.
5662 WaitForEncodedFrame(2);
5663
5664 // Trigger HD "singlecast"
5665 video_encoder_config.simulcast_layers[0].active = false;
5666 video_encoder_config.simulcast_layers[1].active = false;
5667 video_encoder_config.simulcast_layers[2].active = true;
5668
5669 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5670 kMaxPayloadLength);
5671 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5672
5673 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5674 // Frame should be dropped because of initial frame drop.
5675 ExpectDroppedFrame();
5676
5677 // Expect the sink_wants to specify a scaled frame.
5678 EXPECT_TRUE_WAIT(
5679 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5680 video_stream_encoder_->Stop();
5681}
5682
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005683TEST_F(VideoStreamEncoderTest, InitialFrameDropActivatesWhenSVCLayersChange) {
Asa Persson606d3cb2021-10-04 10:07:11 +02005684 const DataRate kLowTargetBitrate = DataRate::KilobitsPerSec(400);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005685 // Set simulcast.
5686 ResetEncoder("VP9", 1, 1, 3, false);
5687 fake_encoder_.SetQualityScaling(true);
5688 const int kWidth = 1280;
5689 const int kHeight = 720;
5690 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005691 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005692 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth, kHeight));
5693 // Frame should not be dropped.
5694 WaitForEncodedFrame(1);
5695
5696 // Trigger QVGA "singlecast"
5697 // Update the config.
5698 VideoEncoderConfig video_encoder_config;
5699 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5700 &video_encoder_config);
5701 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5702 vp9_settings.numberOfSpatialLayers = 3;
5703 // Since only one layer is active - automatic resize should be enabled.
5704 vp9_settings.automaticResizeOn = true;
5705 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005706 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005707 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005708 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005709 video_encoder_config.content_type =
5710 VideoEncoderConfig::ContentType::kRealtimeVideo;
Artem Titovab30d722021-07-27 16:22:11 +02005711 // Currently simulcast layers `active` flags are used to inidicate
Ilya Nikolaevskiycde4a9f2020-11-27 14:06:08 +01005712 // which SVC layers are active.
5713 video_encoder_config.simulcast_layers.resize(3);
5714
5715 video_encoder_config.simulcast_layers[0].active = true;
5716 video_encoder_config.simulcast_layers[1].active = false;
5717 video_encoder_config.simulcast_layers[2].active = false;
5718
5719 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5720 kMaxPayloadLength);
5721 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5722
5723 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth, kHeight));
5724 // Frame should not be dropped.
5725 WaitForEncodedFrame(2);
5726
5727 // Trigger HD "singlecast"
5728 video_encoder_config.simulcast_layers[0].active = false;
5729 video_encoder_config.simulcast_layers[1].active = false;
5730 video_encoder_config.simulcast_layers[2].active = true;
5731
5732 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5733 kMaxPayloadLength);
5734 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5735
5736 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
5737 // Frame should be dropped because of initial frame drop.
5738 ExpectDroppedFrame();
5739
5740 // Expect the sink_wants to specify a scaled frame.
5741 EXPECT_TRUE_WAIT(
5742 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
5743 video_stream_encoder_->Stop();
5744}
5745
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005746TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005747 EncoderMaxAndMinBitratesUsedIfMiddleStreamActive) {
5748 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits270p(
5749 480 * 270, 34 * 1000, 12 * 1000, 1234 * 1000);
5750 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits360p(
5751 640 * 360, 43 * 1000, 21 * 1000, 2345 * 1000);
5752 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5753 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5754 fake_encoder_.SetResolutionBitrateLimits(
5755 {kEncoderLimits270p, kEncoderLimits360p, kEncoderLimits720p});
5756
5757 VideoEncoderConfig video_encoder_config;
5758 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5759 &video_encoder_config);
5760 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5761 vp9_settings.numberOfSpatialLayers = 3;
5762 // Since only one layer is active - automatic resize should be enabled.
5763 vp9_settings.automaticResizeOn = true;
5764 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005765 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005766 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005767 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005768 video_encoder_config.content_type =
5769 VideoEncoderConfig::ContentType::kRealtimeVideo;
5770 // Simulcast layers are used to indicate which spatial layers are active.
5771 video_encoder_config.simulcast_layers.resize(3);
5772 video_encoder_config.simulcast_layers[0].active = false;
5773 video_encoder_config.simulcast_layers[1].active = true;
5774 video_encoder_config.simulcast_layers[2].active = false;
5775
5776 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5777 kMaxPayloadLength);
5778 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5779
5780 // The encoder bitrate limits for 360p should be used.
5781 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5782 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005783 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5784 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5785 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5786 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5787 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5788 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005789 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005790 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005791 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits360p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005792 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005793
5794 // The encoder bitrate limits for 270p should be used.
5795 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5796 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005797 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5798 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5799 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5800 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5801 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5802 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005803 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005804 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005805 EXPECT_EQ(static_cast<uint32_t>(kEncoderLimits270p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005806 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005807
5808 video_stream_encoder_->Stop();
5809}
5810
5811TEST_F(VideoStreamEncoderTest,
Åsa Persson258e9892021-02-25 10:39:51 +01005812 DefaultMaxAndMinBitratesUsedIfMiddleStreamActive) {
5813 VideoEncoderConfig video_encoder_config;
5814 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5815 &video_encoder_config);
5816 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5817 vp9_settings.numberOfSpatialLayers = 3;
5818 // Since only one layer is active - automatic resize should be enabled.
5819 vp9_settings.automaticResizeOn = true;
5820 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005821 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005822 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005823 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005824 video_encoder_config.content_type =
5825 VideoEncoderConfig::ContentType::kRealtimeVideo;
5826 // Simulcast layers are used to indicate which spatial layers are active.
5827 video_encoder_config.simulcast_layers.resize(3);
5828 video_encoder_config.simulcast_layers[0].active = false;
5829 video_encoder_config.simulcast_layers[1].active = true;
5830 video_encoder_config.simulcast_layers[2].active = false;
5831
5832 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5833 kMaxPayloadLength);
5834 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5835
5836 // The default bitrate limits for 360p should be used.
5837 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005838 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5839 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005840 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5841 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005842 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5843 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5844 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5845 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5846 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5847 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005848 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005849 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005850 EXPECT_EQ(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005851 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005852
5853 // The default bitrate limits for 270p should be used.
5854 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits270p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005855 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5856 kVideoCodecVP9, 480 * 270);
Åsa Persson258e9892021-02-25 10:39:51 +01005857 video_source_.IncomingCapturedFrame(CreateFrame(2, 960, 540));
5858 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005859 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5860 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5861 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5862 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5863 EXPECT_EQ(480, fake_encoder_.config().spatialLayers[0].width);
5864 EXPECT_EQ(270, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005865 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005866 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005867 EXPECT_EQ(static_cast<uint32_t>(kLimits270p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005868 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005869
5870 video_stream_encoder_->Stop();
5871}
5872
5873TEST_F(VideoStreamEncoderTest, DefaultMaxAndMinBitratesNotUsedIfDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01005874 webrtc::test::ScopedKeyValueConfig field_trials(
5875 field_trials_, "WebRTC-DefaultBitrateLimitsKillSwitch/Enabled/");
Åsa Persson258e9892021-02-25 10:39:51 +01005876 VideoEncoderConfig video_encoder_config;
5877 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5878 &video_encoder_config);
5879 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5880 vp9_settings.numberOfSpatialLayers = 3;
5881 // Since only one layer is active - automatic resize should be enabled.
5882 vp9_settings.automaticResizeOn = true;
5883 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005884 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Persson258e9892021-02-25 10:39:51 +01005885 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005886 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Persson258e9892021-02-25 10:39:51 +01005887 video_encoder_config.content_type =
5888 VideoEncoderConfig::ContentType::kRealtimeVideo;
5889 // Simulcast layers are used to indicate which spatial layers are active.
5890 video_encoder_config.simulcast_layers.resize(3);
5891 video_encoder_config.simulcast_layers[0].active = false;
5892 video_encoder_config.simulcast_layers[1].active = true;
5893 video_encoder_config.simulcast_layers[2].active = false;
5894
5895 // Reset encoder for field trials to take effect.
5896 ConfigureEncoder(video_encoder_config.Copy());
5897
5898 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5899 kMaxPayloadLength);
5900 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5901
5902 // The default bitrate limits for 360p should not be used.
5903 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits360p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005904 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5905 kVideoCodecVP9, 640 * 360);
Åsa Persson258e9892021-02-25 10:39:51 +01005906 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5907 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005908 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5909 EXPECT_EQ(fake_encoder_.config().codecType, kVideoCodecVP9);
5910 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 2);
5911 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5912 EXPECT_EQ(640, fake_encoder_.config().spatialLayers[0].width);
5913 EXPECT_EQ(360, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005914 EXPECT_NE(static_cast<uint32_t>(kLimits360p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005915 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005916
5917 video_stream_encoder_->Stop();
5918}
5919
5920TEST_F(VideoStreamEncoderTest, SinglecastBitrateLimitsNotUsedForOneStream) {
5921 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
5922 /*num_spatial_layers=*/1, /*screenshare=*/false);
5923
5924 // The default singlecast bitrate limits for 720p should not be used.
5925 const absl::optional<VideoEncoder::ResolutionBitrateLimits> kLimits720p =
Sergey Silkina86b29b2021-03-05 13:29:19 +01005926 EncoderInfoSettings::GetDefaultSinglecastBitrateLimitsForResolution(
5927 kVideoCodecVP9, 1280 * 720);
Åsa Persson258e9892021-02-25 10:39:51 +01005928 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5929 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005930 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5931 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5932 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 1);
5933 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5934 EXPECT_EQ(1280, fake_encoder_.config().spatialLayers[0].width);
5935 EXPECT_EQ(720, fake_encoder_.config().spatialLayers[0].height);
Åsa Persson258e9892021-02-25 10:39:51 +01005936 EXPECT_NE(static_cast<uint32_t>(kLimits720p->max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005937 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Persson258e9892021-02-25 10:39:51 +01005938
5939 video_stream_encoder_->Stop();
5940}
5941
5942TEST_F(VideoStreamEncoderTest,
Åsa Perssonc91c4232021-02-01 09:20:05 +01005943 EncoderMaxAndMinBitratesNotUsedIfLowestStreamActive) {
5944 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits180p(
5945 320 * 180, 34 * 1000, 12 * 1000, 1234 * 1000);
5946 const VideoEncoder::ResolutionBitrateLimits kEncoderLimits720p(
5947 1280 * 720, 54 * 1000, 31 * 1000, 2500 * 1000);
5948 fake_encoder_.SetResolutionBitrateLimits(
5949 {kEncoderLimits180p, kEncoderLimits720p});
5950
5951 VideoEncoderConfig video_encoder_config;
5952 test::FillEncoderConfiguration(PayloadStringToCodecType("VP9"), 1,
5953 &video_encoder_config);
5954 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
5955 vp9_settings.numberOfSpatialLayers = 3;
5956 // Since only one layer is active - automatic resize should be enabled.
5957 vp9_settings.automaticResizeOn = true;
5958 video_encoder_config.encoder_specific_settings =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02005959 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
Åsa Perssonc91c4232021-02-01 09:20:05 +01005960 vp9_settings);
Asa Persson606d3cb2021-10-04 10:07:11 +02005961 video_encoder_config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Åsa Perssonc91c4232021-02-01 09:20:05 +01005962 video_encoder_config.content_type =
5963 VideoEncoderConfig::ContentType::kRealtimeVideo;
5964 // Simulcast layers are used to indicate which spatial layers are active.
5965 video_encoder_config.simulcast_layers.resize(3);
5966 video_encoder_config.simulcast_layers[0].active = true;
5967 video_encoder_config.simulcast_layers[1].active = false;
5968 video_encoder_config.simulcast_layers[2].active = false;
5969
5970 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
5971 kMaxPayloadLength);
5972 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
5973
5974 // Limits not applied on lowest stream, limits for 180p should not be used.
5975 video_source_.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
5976 EXPECT_FALSE(WaitForFrame(1000));
Asa Persson606d3cb2021-10-04 10:07:11 +02005977 EXPECT_EQ(fake_encoder_.config().numberOfSimulcastStreams, 1);
5978 EXPECT_EQ(fake_encoder_.config().codecType, VideoCodecType::kVideoCodecVP9);
5979 EXPECT_EQ(fake_encoder_.config().VP9().numberOfSpatialLayers, 3);
5980 EXPECT_TRUE(fake_encoder_.config().spatialLayers[0].active);
5981 EXPECT_EQ(320, fake_encoder_.config().spatialLayers[0].width);
5982 EXPECT_EQ(180, fake_encoder_.config().spatialLayers[0].height);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005983 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.min_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005984 fake_encoder_.config().spatialLayers[0].minBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005985 EXPECT_NE(static_cast<uint32_t>(kEncoderLimits180p.max_bitrate_bps),
Asa Persson606d3cb2021-10-04 10:07:11 +02005986 fake_encoder_.config().spatialLayers[0].maxBitrate * 1000);
Åsa Perssonc91c4232021-02-01 09:20:05 +01005987
5988 video_stream_encoder_->Stop();
5989}
5990
5991TEST_F(VideoStreamEncoderTest,
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005992 InitialFrameDropActivatesWhenResolutionIncreases) {
5993 const int kWidth = 640;
5994 const int kHeight = 360;
5995
5996 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02005997 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01005998 video_source_.IncomingCapturedFrame(CreateFrame(1, kWidth / 2, kHeight / 2));
5999 // Frame should not be dropped.
6000 WaitForEncodedFrame(1);
6001
6002 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006003 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006004 video_source_.IncomingCapturedFrame(CreateFrame(2, kWidth / 2, kHeight / 2));
6005 // Frame should not be dropped, bitrate not too low for frame.
6006 WaitForEncodedFrame(2);
6007
6008 // Incoming resolution increases.
6009 video_source_.IncomingCapturedFrame(CreateFrame(3, kWidth, kHeight));
6010 // Expect to drop this frame, bitrate too low for frame.
6011 ExpectDroppedFrame();
6012
6013 // Expect the sink_wants to specify a scaled frame.
6014 EXPECT_TRUE_WAIT(
6015 video_source_.sink_wants().max_pixel_count < kWidth * kHeight, 5000);
6016 video_stream_encoder_->Stop();
6017}
6018
6019TEST_F(VideoStreamEncoderTest, InitialFrameDropIsNotReactivatedWhenAdaptingUp) {
6020 const int kWidth = 640;
6021 const int kHeight = 360;
6022 // So that quality scaling doesn't happen by itself.
6023 fake_encoder_.SetQp(kQpHigh);
6024
6025 AdaptingFrameForwarder source(&time_controller_);
6026 source.set_adaptation_enabled(true);
6027 video_stream_encoder_->SetSource(
6028 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
6029
6030 int timestamp = 1;
6031
6032 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006033 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006034 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6035 WaitForEncodedFrame(timestamp);
6036 timestamp += 9000;
6037 // Long pause to disable all first BWE drop logic.
6038 AdvanceTime(TimeDelta::Millis(1000));
6039
6040 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006041 kLowTargetBitrate, kLowTargetBitrate, kLowTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy84bc3482020-11-26 16:58:47 +01006042 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6043 // Not dropped frame, as initial frame drop is disabled by now.
6044 WaitForEncodedFrame(timestamp);
6045 timestamp += 9000;
6046 AdvanceTime(TimeDelta::Millis(100));
6047
6048 // Quality adaptation down.
6049 video_stream_encoder_->TriggerQualityLow();
6050
6051 // Adaptation has an effect.
6052 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6053 5000);
6054
6055 // Frame isn't dropped as initial frame dropper is disabled.
6056 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6057 WaitForEncodedFrame(timestamp);
6058 timestamp += 9000;
6059 AdvanceTime(TimeDelta::Millis(100));
6060
6061 // Quality adaptation up.
6062 video_stream_encoder_->TriggerQualityHigh();
6063
6064 // Adaptation has an effect.
6065 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count > kWidth * kHeight,
6066 5000);
6067
6068 source.IncomingCapturedFrame(CreateFrame(timestamp, kWidth, kHeight));
6069 // Frame should not be dropped, as initial framedropper is off.
6070 WaitForEncodedFrame(timestamp);
6071
6072 video_stream_encoder_->Stop();
6073}
6074
Åsa Persson7f354f82021-02-04 15:52:15 +01006075TEST_F(VideoStreamEncoderTest,
6076 FrameDroppedWhenResolutionIncreasesAndLinkAllocationIsLow) {
6077 const int kMinStartBps360p = 222000;
6078 fake_encoder_.SetResolutionBitrateLimits(
6079 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6080 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6081 800000)});
6082
6083 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6084 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6085 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6086 DataRate::BitsPerSec(kMinStartBps360p - 1), // link_allocation
6087 0, 0, 0);
6088 // Frame should not be dropped, bitrate not too low for frame.
6089 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6090 WaitForEncodedFrame(1);
6091
6092 // Incoming resolution increases, initial frame drop activates.
6093 // Frame should be dropped, link allocation too low for frame.
6094 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6095 ExpectDroppedFrame();
6096
6097 // Expect sink_wants to specify a scaled frame.
6098 EXPECT_TRUE_WAIT(video_source_.sink_wants().max_pixel_count < 640 * 360,
6099 5000);
6100 video_stream_encoder_->Stop();
6101}
6102
6103TEST_F(VideoStreamEncoderTest,
6104 FrameNotDroppedWhenResolutionIncreasesAndLinkAllocationIsHigh) {
6105 const int kMinStartBps360p = 222000;
6106 fake_encoder_.SetResolutionBitrateLimits(
6107 {VideoEncoder::ResolutionBitrateLimits(320 * 180, 0, 30000, 400000),
6108 VideoEncoder::ResolutionBitrateLimits(640 * 360, kMinStartBps360p, 30000,
6109 800000)});
6110
6111 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6112 DataRate::BitsPerSec(kMinStartBps360p - 1), // target_bitrate
6113 DataRate::BitsPerSec(kMinStartBps360p - 1), // stable_target_bitrate
6114 DataRate::BitsPerSec(kMinStartBps360p), // link_allocation
6115 0, 0, 0);
6116 // Frame should not be dropped, bitrate not too low for frame.
6117 video_source_.IncomingCapturedFrame(CreateFrame(1, 320, 180));
6118 WaitForEncodedFrame(1);
6119
6120 // Incoming resolution increases, initial frame drop activates.
6121 // Frame should be dropped, link allocation not too low for frame.
6122 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 360));
6123 WaitForEncodedFrame(2);
6124
6125 video_stream_encoder_->Stop();
6126}
6127
Åsa Perssone644a032019-11-08 15:56:00 +01006128TEST_F(VideoStreamEncoderTest, RampsUpInQualityWhenBwIsHigh) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006129 webrtc::test::ScopedKeyValueConfig field_trials(
6130 field_trials_,
Åsa Persson06defc42021-09-10 15:28:48 +02006131 "WebRTC-Video-QualityRampupSettings/"
6132 "min_pixels:921600,min_duration_ms:2000/");
6133
6134 const int kWidth = 1280;
6135 const int kHeight = 720;
6136 const int kFps = 10;
6137 max_framerate_ = kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006138
6139 // Reset encoder for field trials to take effect.
6140 VideoEncoderConfig config = video_encoder_config_.Copy();
Asa Persson606d3cb2021-10-04 10:07:11 +02006141 config.max_bitrate_bps = kTargetBitrate.bps();
Evan Shrubsoledff79252020-04-16 11:34:32 +02006142 DataRate max_bitrate = DataRate::BitsPerSec(config.max_bitrate_bps);
Åsa Perssone644a032019-11-08 15:56:00 +01006143 ConfigureEncoder(std::move(config));
6144 fake_encoder_.SetQp(kQpLow);
6145
6146 // Enable MAINTAIN_FRAMERATE preference.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006147 AdaptingFrameForwarder source(&time_controller_);
Åsa Perssone644a032019-11-08 15:56:00 +01006148 source.set_adaptation_enabled(true);
6149 video_stream_encoder_->SetSource(&source,
6150 DegradationPreference::MAINTAIN_FRAMERATE);
6151
6152 // Start at low bitrate.
Asa Persson606d3cb2021-10-04 10:07:11 +02006153 const DataRate kLowBitrate = DataRate::KilobitsPerSec(200);
Henrik Boström381d1092020-05-12 18:49:07 +02006154 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006155 kLowBitrate, kLowBitrate, kLowBitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006156
6157 // Expect first frame to be dropped and resolution to be limited.
Åsa Persson06defc42021-09-10 15:28:48 +02006158 const int64_t kFrameIntervalMs = 1000 / kFps;
Åsa Perssone644a032019-11-08 15:56:00 +01006159 int64_t timestamp_ms = kFrameIntervalMs;
6160 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6161 ExpectDroppedFrame();
Henrik Boström2671dac2020-05-19 16:29:09 +02006162 EXPECT_TRUE_WAIT(source.sink_wants().max_pixel_count < kWidth * kHeight,
6163 5000);
Åsa Perssone644a032019-11-08 15:56:00 +01006164
6165 // Increase bitrate to encoder max.
Henrik Boström381d1092020-05-12 18:49:07 +02006166 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
6167 max_bitrate, max_bitrate, max_bitrate, 0, 0, 0);
Åsa Perssone644a032019-11-08 15:56:00 +01006168
Artem Titovab30d722021-07-27 16:22:11 +02006169 // Insert frames and advance `min_duration_ms`.
Åsa Persson06defc42021-09-10 15:28:48 +02006170 const int64_t start_bw_high_ms = CurrentTimeMs();
Åsa Perssone644a032019-11-08 15:56:00 +01006171 for (size_t i = 1; i <= 10; i++) {
6172 timestamp_ms += kFrameIntervalMs;
6173 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6174 WaitForEncodedFrame(timestamp_ms);
6175 }
Åsa Persson06defc42021-09-10 15:28:48 +02006176
6177 // Advance to `min_duration_ms` - 1, frame should not trigger high BW.
6178 int64_t elapsed_bw_high_ms = CurrentTimeMs() - start_bw_high_ms;
6179 AdvanceTime(TimeDelta::Millis(2000 - elapsed_bw_high_ms - 1));
6180 timestamp_ms += kFrameIntervalMs;
6181 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6182 WaitForEncodedFrame(timestamp_ms);
Åsa Perssone644a032019-11-08 15:56:00 +01006183 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6184 EXPECT_LT(source.sink_wants().max_pixel_count, kWidth * kHeight);
6185
Åsa Persson06defc42021-09-10 15:28:48 +02006186 // Frame should trigger high BW and release quality limitation.
Åsa Perssone644a032019-11-08 15:56:00 +01006187 timestamp_ms += kFrameIntervalMs;
6188 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6189 WaitForEncodedFrame(timestamp_ms);
Henrik Boström381d1092020-05-12 18:49:07 +02006190 // The ramp-up code involves the adaptation queue, give it time to execute.
6191 // TODO(hbos): Can we await an appropriate event instead?
Evan Shrubsole85728412020-08-25 13:12:12 +02006192 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006193 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
Åsa Perssone644a032019-11-08 15:56:00 +01006194
6195 // Frame should not be adapted.
6196 timestamp_ms += kFrameIntervalMs;
6197 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6198 WaitForEncodedFrame(kWidth, kHeight);
6199 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6200
6201 video_stream_encoder_->Stop();
6202}
6203
mflodmancc3d4422017-08-03 08:27:51 -07006204TEST_F(VideoStreamEncoderTest,
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006205 QualityScalerAdaptationsRemovedWhenQualityScalingDisabled) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01006206 webrtc::test::ScopedKeyValueConfig field_trials(
6207 field_trials_, "WebRTC-Video-QualityScaling/Disabled/");
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006208 AdaptingFrameForwarder source(&time_controller_);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006209 source.set_adaptation_enabled(true);
6210 video_stream_encoder_->SetSource(&source,
6211 DegradationPreference::MAINTAIN_FRAMERATE);
6212 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006213 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006214 fake_encoder_.SetQp(kQpHigh + 1);
6215 const int kWidth = 1280;
6216 const int kHeight = 720;
6217 const int64_t kFrameIntervalMs = 100;
6218 int64_t timestamp_ms = kFrameIntervalMs;
6219 for (size_t i = 1; i <= 100; i++) {
6220 timestamp_ms += kFrameIntervalMs;
6221 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6222 WaitForEncodedFrame(timestamp_ms);
6223 }
6224 // Wait for QualityScaler, which will wait for 2000*2.5 ms until checking QP
6225 // for the first time.
6226 // TODO(eshr): We should avoid these waits by using threads with simulated
6227 // time.
6228 EXPECT_TRUE_WAIT(stats_proxy_->GetStats().bw_limited_resolution,
6229 2000 * 2.5 * 2);
6230 timestamp_ms += kFrameIntervalMs;
6231 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6232 WaitForEncodedFrame(timestamp_ms);
6233 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
6234 EXPECT_THAT(source.sink_wants(), WantsMaxPixels(Lt(kWidth * kHeight)));
6235 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6236
6237 // Disable Quality scaling by turning off scaler on the encoder and
6238 // reconfiguring.
6239 fake_encoder_.SetQualityScaling(false);
6240 video_stream_encoder_->ConfigureEncoder(video_encoder_config_.Copy(),
6241 kMaxPayloadLength);
6242 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Markus Handell28c71802021-11-08 10:11:55 +01006243 AdvanceTime(TimeDelta::Zero());
Evan Shrubsole99b0f8d2020-08-25 13:12:28 +02006244 // Since we turned off the quality scaler, the adaptations made by it are
6245 // removed.
6246 EXPECT_THAT(source.sink_wants(), ResolutionMax());
6247 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6248
6249 video_stream_encoder_->Stop();
6250}
6251
6252TEST_F(VideoStreamEncoderTest,
asaperssond0de2952017-04-21 01:47:31 -07006253 ResolutionNotAdaptedForTooSmallFrame_MaintainFramerateMode) {
6254 const int kTooSmallWidth = 10;
6255 const int kTooSmallHeight = 10;
Henrik Boström381d1092020-05-12 18:49:07 +02006256 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006257 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssond0de2952017-04-21 01:47:31 -07006258
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006259 // Enable MAINTAIN_FRAMERATE preference, no initial limitation.
asaperssond0de2952017-04-21 01:47:31 -07006260 test::FrameForwarder source;
mflodmancc3d4422017-08-03 08:27:51 -07006261 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006262 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006263 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssond0de2952017-04-21 01:47:31 -07006264 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6265
6266 // Trigger adapt down, too small frame, expect no change.
6267 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006268 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006269 video_stream_encoder_->TriggerCpuOveruse();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006270 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssond0de2952017-04-21 01:47:31 -07006271 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6272 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6273
mflodmancc3d4422017-08-03 08:27:51 -07006274 video_stream_encoder_->Stop();
asaperssond0de2952017-04-21 01:47:31 -07006275}
6276
mflodmancc3d4422017-08-03 08:27:51 -07006277TEST_F(VideoStreamEncoderTest,
6278 ResolutionNotAdaptedForTooSmallFrame_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006279 const int kTooSmallWidth = 10;
6280 const int kTooSmallHeight = 10;
6281 const int kFpsLimit = 7;
Henrik Boström381d1092020-05-12 18:49:07 +02006282 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006283 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006284
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006285 // Enable BALANCED preference, no initial limitation.
asaperssonf7e294d2017-06-13 23:25:22 -07006286 test::FrameForwarder source;
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006287 video_stream_encoder_->SetSource(&source,
6288 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006289 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
asaperssonf7e294d2017-06-13 23:25:22 -07006290 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6291 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6292
6293 // Trigger adapt down, expect limited framerate.
6294 source.IncomingCapturedFrame(CreateFrame(1, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006295 WaitForEncodedFrame(1);
mflodmancc3d4422017-08-03 08:27:51 -07006296 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006297 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006298 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6299 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6300 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6301
6302 // Trigger adapt down, too small frame, expect no change.
6303 source.IncomingCapturedFrame(CreateFrame(2, kTooSmallWidth, kTooSmallHeight));
sprang4847ae62017-06-27 07:06:52 -07006304 WaitForEncodedFrame(2);
mflodmancc3d4422017-08-03 08:27:51 -07006305 video_stream_encoder_->TriggerQualityLow();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006306 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006307 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6308 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6309 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6310
mflodmancc3d4422017-08-03 08:27:51 -07006311 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006312}
6313
mflodmancc3d4422017-08-03 08:27:51 -07006314TEST_F(VideoStreamEncoderTest, FailingInitEncodeDoesntCauseCrash) {
asapersson02465b82017-04-10 01:12:52 -07006315 fake_encoder_.ForceInitEncodeFailure(true);
Henrik Boström381d1092020-05-12 18:49:07 +02006316 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006317 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möllerf1338562018-04-26 09:51:47 +02006318 ResetEncoder("VP8", 2, 1, 1, false);
asapersson02465b82017-04-10 01:12:52 -07006319 const int kFrameWidth = 1280;
6320 const int kFrameHeight = 720;
6321 video_source_.IncomingCapturedFrame(
6322 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006323 ExpectDroppedFrame();
mflodmancc3d4422017-08-03 08:27:51 -07006324 video_stream_encoder_->Stop();
asapersson02465b82017-04-10 01:12:52 -07006325}
6326
sprangb1ca0732017-02-01 08:38:12 -08006327// TODO(sprang): Extend this with fps throttling and any "balanced" extensions.
mflodmancc3d4422017-08-03 08:27:51 -07006328TEST_F(VideoStreamEncoderTest,
6329 AdaptsResolutionOnOveruse_MaintainFramerateMode) {
Henrik Boström381d1092020-05-12 18:49:07 +02006330 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006331 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprangb1ca0732017-02-01 08:38:12 -08006332
6333 const int kFrameWidth = 1280;
6334 const int kFrameHeight = 720;
6335 // Enabled default VideoAdapter downscaling. First step is 3/4, not 3/5 as
mflodmancc3d4422017-08-03 08:27:51 -07006336 // requested by
6337 // VideoStreamEncoder::VideoSourceProxy::RequestResolutionLowerThan().
sprangb1ca0732017-02-01 08:38:12 -08006338 video_source_.set_adaptation_enabled(true);
6339
6340 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006341 CreateFrame(1 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006342 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006343
6344 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006345 video_stream_encoder_->TriggerCpuOveruse();
sprangb1ca0732017-02-01 08:38:12 -08006346 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006347 CreateFrame(2 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006348 WaitForEncodedFrame((kFrameWidth * 3) / 4, (kFrameHeight * 3) / 4);
sprangb1ca0732017-02-01 08:38:12 -08006349
asaperssonfab67072017-04-04 05:51:49 -07006350 // Trigger CPU normal use, return to original resolution.
Henrik Boström91aa7322020-04-28 12:24:33 +02006351 video_stream_encoder_->TriggerCpuUnderuse();
sprangb1ca0732017-02-01 08:38:12 -08006352 video_source_.IncomingCapturedFrame(
Åsa Persson8c1bf952018-09-13 10:42:19 +02006353 CreateFrame(3 * kFrameIntervalMs, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006354 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
sprangb1ca0732017-02-01 08:38:12 -08006355
mflodmancc3d4422017-08-03 08:27:51 -07006356 video_stream_encoder_->Stop();
sprangb1ca0732017-02-01 08:38:12 -08006357}
sprangfe627f32017-03-29 08:24:59 -07006358
mflodmancc3d4422017-08-03 08:27:51 -07006359TEST_F(VideoStreamEncoderTest,
6360 AdaptsFramerateOnOveruse_MaintainResolutionMode) {
sprangc5d62e22017-04-02 23:53:04 -07006361 const int kFrameWidth = 1280;
6362 const int kFrameHeight = 720;
sprangc5d62e22017-04-02 23:53:04 -07006363
Henrik Boström381d1092020-05-12 18:49:07 +02006364 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006365 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006366 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006367 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006368 video_source_.set_adaptation_enabled(true);
6369
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006370 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006371
6372 video_source_.IncomingCapturedFrame(
6373 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006374 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006375
6376 // Try to trigger overuse. No fps estimate available => no effect.
mflodmancc3d4422017-08-03 08:27:51 -07006377 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006378
6379 // Insert frames for one second to get a stable estimate.
sprang4847ae62017-06-27 07:06:52 -07006380 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006381 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006382 video_source_.IncomingCapturedFrame(
6383 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006384 WaitForEncodedFrame(timestamp_ms);
sprangc5d62e22017-04-02 23:53:04 -07006385 }
6386
6387 // Trigger CPU overuse, reduce framerate by 2/3.
mflodmancc3d4422017-08-03 08:27:51 -07006388 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006389 int num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006390 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006391 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006392 video_source_.IncomingCapturedFrame(
6393 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006394 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006395 ++num_frames_dropped;
6396 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006397 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006398 }
6399 }
6400
sprang4847ae62017-06-27 07:06:52 -07006401 // Add some slack to account for frames dropped by the frame dropper.
6402 const int kErrorMargin = 1;
6403 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006404 kErrorMargin);
6405
6406 // Trigger CPU overuse, reduce framerate by 2/3 again.
mflodmancc3d4422017-08-03 08:27:51 -07006407 video_stream_encoder_->TriggerCpuOveruse();
sprangc5d62e22017-04-02 23:53:04 -07006408 num_frames_dropped = 0;
Åsa Persson8c1bf952018-09-13 10:42:19 +02006409 for (int i = 0; i <= max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006410 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006411 video_source_.IncomingCapturedFrame(
6412 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006413 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006414 ++num_frames_dropped;
6415 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006416 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006417 }
6418 }
sprang4847ae62017-06-27 07:06:52 -07006419 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 4 / 9),
sprangc5d62e22017-04-02 23:53:04 -07006420 kErrorMargin);
6421
6422 // Go back up one step.
Henrik Boström91aa7322020-04-28 12:24:33 +02006423 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006424 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006425 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006426 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006427 video_source_.IncomingCapturedFrame(
6428 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006429 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006430 ++num_frames_dropped;
6431 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006432 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006433 }
6434 }
sprang4847ae62017-06-27 07:06:52 -07006435 EXPECT_NEAR(num_frames_dropped, max_framerate_ - (max_framerate_ * 2 / 3),
sprangc5d62e22017-04-02 23:53:04 -07006436 kErrorMargin);
6437
6438 // Go back up to original mode.
Henrik Boström91aa7322020-04-28 12:24:33 +02006439 video_stream_encoder_->TriggerCpuUnderuse();
sprangc5d62e22017-04-02 23:53:04 -07006440 num_frames_dropped = 0;
sprang4847ae62017-06-27 07:06:52 -07006441 for (int i = 0; i < max_framerate_; ++i) {
sprangc5d62e22017-04-02 23:53:04 -07006442 timestamp_ms += kFrameIntervalMs;
sprangc5d62e22017-04-02 23:53:04 -07006443 video_source_.IncomingCapturedFrame(
6444 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006445 if (!WaitForFrame(kFrameTimeoutMs)) {
sprangc5d62e22017-04-02 23:53:04 -07006446 ++num_frames_dropped;
6447 } else {
Åsa Perssonc74d8da2017-12-04 14:13:56 +01006448 sink_.CheckLastFrameSizeMatches(kFrameWidth, kFrameHeight);
sprangc5d62e22017-04-02 23:53:04 -07006449 }
6450 }
6451 EXPECT_NEAR(num_frames_dropped, 0, kErrorMargin);
6452
mflodmancc3d4422017-08-03 08:27:51 -07006453 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006454}
6455
mflodmancc3d4422017-08-03 08:27:51 -07006456TEST_F(VideoStreamEncoderTest, DoesntAdaptDownPastMinFramerate) {
sprangc5d62e22017-04-02 23:53:04 -07006457 const int kFramerateFps = 5;
6458 const int kFrameIntervalMs = rtc::kNumMillisecsPerSec / kFramerateFps;
sprangc5d62e22017-04-02 23:53:04 -07006459 const int kFrameWidth = 1280;
6460 const int kFrameHeight = 720;
6461
sprang4847ae62017-06-27 07:06:52 -07006462 // Reconfigure encoder with two temporal layers and screensharing, which will
6463 // disable frame dropping and make testing easier.
Niels Möllerf1338562018-04-26 09:51:47 +02006464 ResetEncoder("VP8", 1, 2, 1, true);
sprang4847ae62017-06-27 07:06:52 -07006465
Henrik Boström381d1092020-05-12 18:49:07 +02006466 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006467 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07006468 video_stream_encoder_->SetSource(
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006469 &video_source_, webrtc::DegradationPreference::MAINTAIN_RESOLUTION);
sprangc5d62e22017-04-02 23:53:04 -07006470 video_source_.set_adaptation_enabled(true);
6471
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006472 int64_t timestamp_ms = CurrentTimeMs();
sprangc5d62e22017-04-02 23:53:04 -07006473
6474 // Trigger overuse as much as we can.
Jonathan Yubc771b72017-12-08 17:04:29 -08006475 rtc::VideoSinkWants last_wants;
6476 do {
6477 last_wants = video_source_.sink_wants();
6478
sprangc5d62e22017-04-02 23:53:04 -07006479 // Insert frames to get a new fps estimate...
6480 for (int j = 0; j < kFramerateFps; ++j) {
6481 video_source_.IncomingCapturedFrame(
6482 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
Jonathan Yubc771b72017-12-08 17:04:29 -08006483 if (video_source_.last_sent_width()) {
6484 sink_.WaitForEncodedFrame(timestamp_ms);
6485 }
sprangc5d62e22017-04-02 23:53:04 -07006486 timestamp_ms += kFrameIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006487 AdvanceTime(TimeDelta::Millis(kFrameIntervalMs));
sprangc5d62e22017-04-02 23:53:04 -07006488 }
6489 // ...and then try to adapt again.
mflodmancc3d4422017-08-03 08:27:51 -07006490 video_stream_encoder_->TriggerCpuOveruse();
Jonathan Yubc771b72017-12-08 17:04:29 -08006491 } while (video_source_.sink_wants().max_framerate_fps <
6492 last_wants.max_framerate_fps);
sprangc5d62e22017-04-02 23:53:04 -07006493
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006494 EXPECT_THAT(video_source_.sink_wants(),
6495 FpsMatchesResolutionMax(Eq(kMinFramerateFps)));
asaperssonf7e294d2017-06-13 23:25:22 -07006496
mflodmancc3d4422017-08-03 08:27:51 -07006497 video_stream_encoder_->Stop();
sprangc5d62e22017-04-02 23:53:04 -07006498}
asaperssonf7e294d2017-06-13 23:25:22 -07006499
mflodmancc3d4422017-08-03 08:27:51 -07006500TEST_F(VideoStreamEncoderTest,
6501 AdaptsResolutionAndFramerateForLowQuality_BalancedMode) {
asaperssonf7e294d2017-06-13 23:25:22 -07006502 const int kWidth = 1280;
6503 const int kHeight = 720;
6504 const int64_t kFrameIntervalMs = 150;
6505 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006506 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006507 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006508
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006509 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006510 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006511 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006512 video_stream_encoder_->SetSource(&source,
6513 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006514 timestamp_ms += kFrameIntervalMs;
6515 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006516 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006517 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006518 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6519 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6520 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6521
6522 // Trigger adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006523 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006524 timestamp_ms += kFrameIntervalMs;
6525 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006526 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006527 EXPECT_THAT(source.sink_wants(),
6528 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006529 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6530 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6531 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6532
6533 // Trigger adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006534 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006535 timestamp_ms += kFrameIntervalMs;
6536 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006537 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006538 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006539 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6540 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6541 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6542
6543 // Trigger adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006544 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006545 timestamp_ms += kFrameIntervalMs;
6546 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006547 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006548 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006549 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6550 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6551 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6552
6553 // Trigger adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006554 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006555 timestamp_ms += kFrameIntervalMs;
6556 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006557 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006558 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006559 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6560 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6561 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6562
6563 // Restrict bitrate, trigger adapt down, expect reduced fps (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006564 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006565 timestamp_ms += kFrameIntervalMs;
6566 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006567 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006568 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006569 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6570 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6571 EXPECT_EQ(5, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6572
6573 // Trigger adapt down, expect scaled down resolution (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006574 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006575 timestamp_ms += kFrameIntervalMs;
6576 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006577 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006578 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006579 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6580 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6581 EXPECT_EQ(6, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6582
6583 // Trigger adapt down, expect reduced fps (320x180@7fps).
mflodmancc3d4422017-08-03 08:27:51 -07006584 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006585 timestamp_ms += kFrameIntervalMs;
6586 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006587 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006588 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006589 rtc::VideoSinkWants last_wants = source.sink_wants();
6590 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6591 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6592 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6593
6594 // Trigger adapt down, min resolution reached, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006595 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006596 timestamp_ms += kFrameIntervalMs;
6597 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006598 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006599 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(last_wants));
asaperssonf7e294d2017-06-13 23:25:22 -07006600 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6601 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6602 EXPECT_EQ(7, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6603
Åsa Perssonf5f7e8e2021-06-09 22:55:38 +02006604 // Trigger adapt up, expect increased fps (320x180@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006605 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006606 timestamp_ms += kFrameIntervalMs;
6607 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006608 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006609 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006610 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6611 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6612 EXPECT_EQ(8, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6613
6614 // Trigger adapt up, expect upscaled resolution (480x270@10fps).
mflodmancc3d4422017-08-03 08:27:51 -07006615 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006616 timestamp_ms += kFrameIntervalMs;
6617 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006618 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006619 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006620 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6621 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6622 EXPECT_EQ(9, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6623
6624 // Increase bitrate, trigger adapt up, expect increased fps (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006625 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006626 timestamp_ms += kFrameIntervalMs;
6627 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006628 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006629 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006630 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6631 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6632 EXPECT_EQ(10, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6633
6634 // Trigger adapt up, expect upscaled resolution (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006635 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006636 timestamp_ms += kFrameIntervalMs;
6637 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006638 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006639 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006640 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6641 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6642 EXPECT_EQ(11, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6643
6644 // Trigger adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006645 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006646 timestamp_ms += kFrameIntervalMs;
6647 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006648 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006649 EXPECT_THAT(source.sink_wants(), FpsMax());
6650 EXPECT_EQ(source.sink_wants().max_pixel_count,
6651 source.last_wants().max_pixel_count);
asaperssonf7e294d2017-06-13 23:25:22 -07006652 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6653 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6654 EXPECT_EQ(12, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6655
6656 // Trigger adapt up, expect upscaled resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006657 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006658 timestamp_ms += kFrameIntervalMs;
6659 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006660 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006661 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006662 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6663 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6664 EXPECT_EQ(13, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6665
Åsa Persson30ab0152019-08-27 12:22:33 +02006666 // Trigger adapt up, expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006667 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006668 timestamp_ms += kFrameIntervalMs;
6669 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006670 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006671 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006672 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006673 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6674 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6675 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6676
6677 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006678 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006679 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006680 EXPECT_EQ(14, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6681
mflodmancc3d4422017-08-03 08:27:51 -07006682 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006683}
6684
mflodmancc3d4422017-08-03 08:27:51 -07006685TEST_F(VideoStreamEncoderTest, AdaptWithTwoReasonsAndDifferentOrder_Framerate) {
asaperssonf7e294d2017-06-13 23:25:22 -07006686 const int kWidth = 1280;
6687 const int kHeight = 720;
6688 const int64_t kFrameIntervalMs = 150;
6689 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006690 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006691 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006692
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006693 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006694 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006695 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006696 video_stream_encoder_->SetSource(&source,
6697 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006698 timestamp_ms += kFrameIntervalMs;
6699 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006700 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006701 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006702 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6703 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6704 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6705 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6706 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6707 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6708
6709 // Trigger cpu adapt down, expect scaled down resolution (960x540@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006710 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006711 timestamp_ms += kFrameIntervalMs;
6712 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006713 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006714 EXPECT_THAT(source.sink_wants(),
6715 FpsMaxResolutionMatches(Lt(kWidth * kHeight)));
asaperssonf7e294d2017-06-13 23:25:22 -07006716 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6717 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6718 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6719 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6720 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6721 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6722
6723 // Trigger cpu adapt down, expect scaled down resolution (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006724 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006725 timestamp_ms += kFrameIntervalMs;
6726 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006727 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006728 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006729 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6730 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6731 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6732 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6733 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6734 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6735
6736 // Trigger quality adapt down, expect reduced fps (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006737 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006738 timestamp_ms += kFrameIntervalMs;
6739 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006740 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006741 EXPECT_THAT(source.sink_wants(), FpsLtResolutionEq(source.last_wants()));
Evan Shrubsole64469032020-06-11 10:45:29 +02006742 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
asaperssonf7e294d2017-06-13 23:25:22 -07006743 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6744 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6745 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6746 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6747 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6748
Evan Shrubsole64469032020-06-11 10:45:29 +02006749 // Trigger cpu adapt up, expect no change since QP is most limited.
6750 {
6751 // Store current sink wants since we expect no change and if there is no
6752 // change then last_wants() is not updated.
6753 auto previous_sink_wants = source.sink_wants();
6754 video_stream_encoder_->TriggerCpuUnderuse();
6755 timestamp_ms += kFrameIntervalMs;
6756 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6757 WaitForEncodedFrame(timestamp_ms);
6758 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6759 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6760 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6761 }
6762
6763 // Trigger quality adapt up, expect increased fps (640x360@30fps).
6764 video_stream_encoder_->TriggerQualityHigh();
6765 timestamp_ms += kFrameIntervalMs;
6766 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6767 WaitForEncodedFrame(timestamp_ms);
6768 EXPECT_THAT(source.sink_wants(), FpsGtResolutionEq(source.last_wants()));
6769 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6770 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6771 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6772 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6773 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6774 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6775
6776 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6777 // expect increased resolution (960x540@30fps).
6778 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006779 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006780 timestamp_ms += kFrameIntervalMs;
6781 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006782 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole64469032020-06-11 10:45:29 +02006783 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006784 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
6785 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6786 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_resolution);
6787 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6788 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006789 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006790
Evan Shrubsole64469032020-06-11 10:45:29 +02006791 // Trigger quality adapt up and Cpu adapt up since both are most limited,
6792 // expect no restriction (1280x720fps@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006793 video_stream_encoder_->TriggerQualityHigh();
Henrik Boström91aa7322020-04-28 12:24:33 +02006794 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006795 timestamp_ms += kFrameIntervalMs;
6796 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006797 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006798 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionGt(source.last_wants()));
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006799 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006800 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6801 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6802 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6803 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6804 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006805 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006806
6807 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006808 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006809 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006810 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006811 EXPECT_EQ(4, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006812
mflodmancc3d4422017-08-03 08:27:51 -07006813 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006814}
6815
mflodmancc3d4422017-08-03 08:27:51 -07006816TEST_F(VideoStreamEncoderTest,
6817 AdaptWithTwoReasonsAndDifferentOrder_Resolution) {
asaperssonf7e294d2017-06-13 23:25:22 -07006818 const int kWidth = 640;
6819 const int kHeight = 360;
6820 const int kFpsLimit = 15;
6821 const int64_t kFrameIntervalMs = 150;
6822 int64_t timestamp_ms = kFrameIntervalMs;
Henrik Boström381d1092020-05-12 18:49:07 +02006823 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006824 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
asaperssonf7e294d2017-06-13 23:25:22 -07006825
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006826 // Enable BALANCED preference, no initial limitation.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006827 AdaptingFrameForwarder source(&time_controller_);
asaperssonf7e294d2017-06-13 23:25:22 -07006828 source.set_adaptation_enabled(true);
Taylor Brandstetter49fcc102018-05-16 14:20:41 -07006829 video_stream_encoder_->SetSource(&source,
6830 webrtc::DegradationPreference::BALANCED);
asaperssonf7e294d2017-06-13 23:25:22 -07006831 timestamp_ms += kFrameIntervalMs;
6832 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006833 WaitForEncodedFrame(kWidth, kHeight);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006834 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006835 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6836 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6837 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6838 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6839 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6840 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6841
6842 // Trigger cpu adapt down, expect scaled down framerate (640x360@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006843 video_stream_encoder_->TriggerCpuOveruse();
asaperssonf7e294d2017-06-13 23:25:22 -07006844 timestamp_ms += kFrameIntervalMs;
6845 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006846 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006847 EXPECT_THAT(source.sink_wants(), FpsMatchesResolutionMax(Eq(kFpsLimit)));
asaperssonf7e294d2017-06-13 23:25:22 -07006848 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6849 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6850 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6851 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6852 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6853 EXPECT_EQ(0, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6854
6855 // Trigger quality adapt down, expect scaled down resolution (480x270@15fps).
mflodmancc3d4422017-08-03 08:27:51 -07006856 video_stream_encoder_->TriggerQualityLow();
asaperssonf7e294d2017-06-13 23:25:22 -07006857 timestamp_ms += kFrameIntervalMs;
6858 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006859 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006860 EXPECT_THAT(source.sink_wants(), FpsEqResolutionLt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006861 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006862 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
asaperssonf7e294d2017-06-13 23:25:22 -07006863 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6864 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6865 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6866 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6867
Evan Shrubsole64469032020-06-11 10:45:29 +02006868 // Trigger cpu adapt up, expect no change because quality is most limited.
6869 {
6870 auto previous_sink_wants = source.sink_wants();
6871 // Store current sink wants since we expect no change ind if there is no
6872 // change then last__wants() is not updated.
6873 video_stream_encoder_->TriggerCpuUnderuse();
6874 timestamp_ms += kFrameIntervalMs;
6875 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
6876 WaitForEncodedFrame(timestamp_ms);
6877 EXPECT_THAT(source.sink_wants(), FpsEqResolutionEqTo(previous_sink_wants));
6878 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6879 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_quality_adapt_changes);
6880 }
6881
6882 // Trigger quality adapt up, expect upscaled resolution (640x360@15fps).
6883 video_stream_encoder_->TriggerQualityHigh();
asaperssonf7e294d2017-06-13 23:25:22 -07006884 timestamp_ms += kFrameIntervalMs;
6885 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006886 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5fd40602020-05-25 16:19:54 +02006887 EXPECT_THAT(source.sink_wants(), FpsEqResolutionGt(source.last_wants()));
asaperssonf7e294d2017-06-13 23:25:22 -07006888 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6889 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_framerate);
6890 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
Evan Shrubsole64469032020-06-11 10:45:29 +02006891 EXPECT_TRUE(stats_proxy_->GetStats().cpu_limited_framerate);
6892 EXPECT_EQ(1, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
6893 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006894
Evan Shrubsole64469032020-06-11 10:45:29 +02006895 // Trigger quality and cpu adapt up, expect increased fps (640x360@30fps).
mflodmancc3d4422017-08-03 08:27:51 -07006896 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole64469032020-06-11 10:45:29 +02006897 video_stream_encoder_->TriggerCpuUnderuse();
asaperssonf7e294d2017-06-13 23:25:22 -07006898 timestamp_ms += kFrameIntervalMs;
6899 source.IncomingCapturedFrame(CreateFrame(timestamp_ms, kWidth, kHeight));
sprang4847ae62017-06-27 07:06:52 -07006900 WaitForEncodedFrame(timestamp_ms);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006901 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006902 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
6903 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_framerate);
6904 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_resolution);
6905 EXPECT_FALSE(stats_proxy_->GetStats().cpu_limited_framerate);
6906 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006907 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006908
6909 // Trigger adapt up, expect no change.
mflodmancc3d4422017-08-03 08:27:51 -07006910 video_stream_encoder_->TriggerQualityHigh();
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02006911 EXPECT_THAT(source.sink_wants(), FpsMaxResolutionMax());
asaperssonf7e294d2017-06-13 23:25:22 -07006912 EXPECT_EQ(2, stats_proxy_->GetStats().number_of_cpu_adapt_changes);
Evan Shrubsole64469032020-06-11 10:45:29 +02006913 EXPECT_EQ(3, stats_proxy_->GetStats().number_of_quality_adapt_changes);
asaperssonf7e294d2017-06-13 23:25:22 -07006914
mflodmancc3d4422017-08-03 08:27:51 -07006915 video_stream_encoder_->Stop();
asaperssonf7e294d2017-06-13 23:25:22 -07006916}
6917
mflodmancc3d4422017-08-03 08:27:51 -07006918TEST_F(VideoStreamEncoderTest, AcceptsFullHdAdaptedDownSimulcastFrames) {
ilnik6b826ef2017-06-16 06:53:48 -07006919 const int kFrameWidth = 1920;
6920 const int kFrameHeight = 1080;
6921 // 3/4 of 1920.
6922 const int kAdaptedFrameWidth = 1440;
6923 // 3/4 of 1080 rounded down to multiple of 4.
6924 const int kAdaptedFrameHeight = 808;
6925 const int kFramerate = 24;
6926
Henrik Boström381d1092020-05-12 18:49:07 +02006927 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006928 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
ilnik6b826ef2017-06-16 06:53:48 -07006929 // Trigger reconfigure encoder (without resetting the entire instance).
6930 VideoEncoderConfig video_encoder_config;
Åsa Persson17b29b92020-10-17 12:57:58 +02006931 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &video_encoder_config);
6932 video_encoder_config.simulcast_layers[0].max_framerate = kFramerate;
Asa Persson606d3cb2021-10-04 10:07:11 +02006933 video_encoder_config.max_bitrate_bps = kTargetBitrate.bps();
ilnik6b826ef2017-06-16 06:53:48 -07006934 video_encoder_config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02006935 rtc::make_ref_counted<CroppingVideoStreamFactory>();
mflodmancc3d4422017-08-03 08:27:51 -07006936 video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
Niels Möllerf1338562018-04-26 09:51:47 +02006937 kMaxPayloadLength);
mflodmancc3d4422017-08-03 08:27:51 -07006938 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
ilnik6b826ef2017-06-16 06:53:48 -07006939
6940 video_source_.set_adaptation_enabled(true);
6941
6942 video_source_.IncomingCapturedFrame(
6943 CreateFrame(1, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006944 WaitForEncodedFrame(kFrameWidth, kFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006945
6946 // Trigger CPU overuse, downscale by 3/4.
mflodmancc3d4422017-08-03 08:27:51 -07006947 video_stream_encoder_->TriggerCpuOveruse();
ilnik6b826ef2017-06-16 06:53:48 -07006948 video_source_.IncomingCapturedFrame(
6949 CreateFrame(2, kFrameWidth, kFrameHeight));
sprang4847ae62017-06-27 07:06:52 -07006950 WaitForEncodedFrame(kAdaptedFrameWidth, kAdaptedFrameHeight);
ilnik6b826ef2017-06-16 06:53:48 -07006951
mflodmancc3d4422017-08-03 08:27:51 -07006952 video_stream_encoder_->Stop();
ilnik6b826ef2017-06-16 06:53:48 -07006953}
6954
mflodmancc3d4422017-08-03 08:27:51 -07006955TEST_F(VideoStreamEncoderTest, PeriodicallyUpdatesChannelParameters) {
sprang4847ae62017-06-27 07:06:52 -07006956 const int kFrameWidth = 1280;
6957 const int kFrameHeight = 720;
6958 const int kLowFps = 2;
6959 const int kHighFps = 30;
6960
Henrik Boström381d1092020-05-12 18:49:07 +02006961 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006962 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006963
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02006964 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07006965 max_framerate_ = kLowFps;
6966
6967 // Insert 2 seconds of 2fps video.
6968 for (int i = 0; i < kLowFps * 2; ++i) {
6969 video_source_.IncomingCapturedFrame(
6970 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6971 WaitForEncodedFrame(timestamp_ms);
6972 timestamp_ms += 1000 / kLowFps;
6973 }
6974
6975 // Make sure encoder is updated with new target.
Henrik Boström381d1092020-05-12 18:49:07 +02006976 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02006977 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
sprang4847ae62017-06-27 07:06:52 -07006978 video_source_.IncomingCapturedFrame(
6979 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6980 WaitForEncodedFrame(timestamp_ms);
6981 timestamp_ms += 1000 / kLowFps;
6982
6983 EXPECT_EQ(kLowFps, fake_encoder_.GetConfiguredInputFramerate());
6984
6985 // Insert 30fps frames for just a little more than the forced update period.
Niels Möllerfe407b72019-09-10 10:48:48 +02006986 const int kVcmTimerIntervalFrames = (kProcessIntervalMs * kHighFps) / 1000;
sprang4847ae62017-06-27 07:06:52 -07006987 const int kFrameIntervalMs = 1000 / kHighFps;
6988 max_framerate_ = kHighFps;
6989 for (int i = 0; i < kVcmTimerIntervalFrames + 2; ++i) {
6990 video_source_.IncomingCapturedFrame(
6991 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
6992 // Wait for encoded frame, but skip ahead if it doesn't arrive as it might
6993 // be dropped if the encoder hans't been updated with the new higher target
6994 // framerate yet, causing it to overshoot the target bitrate and then
6995 // suffering the wrath of the media optimizer.
6996 TimedWaitForEncodedFrame(timestamp_ms, 2 * kFrameIntervalMs);
6997 timestamp_ms += kFrameIntervalMs;
6998 }
6999
7000 // Don expect correct measurement just yet, but it should be higher than
7001 // before.
7002 EXPECT_GT(fake_encoder_.GetConfiguredInputFramerate(), kLowFps);
7003
mflodmancc3d4422017-08-03 08:27:51 -07007004 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007005}
7006
mflodmancc3d4422017-08-03 08:27:51 -07007007TEST_F(VideoStreamEncoderTest, DoesNotUpdateBitrateAllocationWhenSuspended) {
sprang4847ae62017-06-27 07:06:52 -07007008 const int kFrameWidth = 1280;
7009 const int kFrameHeight = 720;
Per Kjellanderdcef6412020-10-07 15:09:05 +02007010 ResetEncoder("FAKE", 1, 1, 1, false,
Per Kjellanderb03b6c82021-01-03 10:26:03 +01007011 VideoStreamEncoder::BitrateAllocationCallbackType::
Per Kjellanderdcef6412020-10-07 15:09:05 +02007012 kVideoBitrateAllocation);
sprang4847ae62017-06-27 07:06:52 -07007013
Henrik Boström381d1092020-05-12 18:49:07 +02007014 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007015 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
mflodmancc3d4422017-08-03 08:27:51 -07007016 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
sprang4847ae62017-06-27 07:06:52 -07007017
7018 // Insert a first video frame, causes another bitrate update.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007019 int64_t timestamp_ms = CurrentTimeMs();
sprang4847ae62017-06-27 07:06:52 -07007020 video_source_.IncomingCapturedFrame(
7021 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7022 WaitForEncodedFrame(timestamp_ms);
Per Kjellanderdcef6412020-10-07 15:09:05 +02007023 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007024
7025 // Next, simulate video suspension due to pacer queue overrun.
Henrik Boström381d1092020-05-12 18:49:07 +02007026 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007027 DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 1, 0);
sprang4847ae62017-06-27 07:06:52 -07007028
7029 // Skip ahead until a new periodic parameter update should have occured.
Niels Möllerfe407b72019-09-10 10:48:48 +02007030 timestamp_ms += kProcessIntervalMs;
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007031 AdvanceTime(TimeDelta::Millis(kProcessIntervalMs));
sprang4847ae62017-06-27 07:06:52 -07007032
Per Kjellanderdcef6412020-10-07 15:09:05 +02007033 // No more allocations has been made.
sprang4847ae62017-06-27 07:06:52 -07007034 video_source_.IncomingCapturedFrame(
7035 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7036 ExpectDroppedFrame();
Per Kjellanderdcef6412020-10-07 15:09:05 +02007037 EXPECT_EQ(sink_.number_of_bitrate_allocations(), 1);
sprang4847ae62017-06-27 07:06:52 -07007038
mflodmancc3d4422017-08-03 08:27:51 -07007039 video_stream_encoder_->Stop();
sprang4847ae62017-06-27 07:06:52 -07007040}
ilnik6b826ef2017-06-16 06:53:48 -07007041
Niels Möller4db138e2018-04-19 09:04:13 +02007042TEST_F(VideoStreamEncoderTest,
7043 DefaultCpuAdaptationThresholdsForSoftwareEncoder) {
7044 const int kFrameWidth = 1280;
7045 const int kFrameHeight = 720;
7046 const CpuOveruseOptions default_options;
Henrik Boström381d1092020-05-12 18:49:07 +02007047 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007048 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007049 video_source_.IncomingCapturedFrame(
7050 CreateFrame(1, kFrameWidth, kFrameHeight));
7051 WaitForEncodedFrame(1);
7052 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7053 .low_encode_usage_threshold_percent,
7054 default_options.low_encode_usage_threshold_percent);
7055 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7056 .high_encode_usage_threshold_percent,
7057 default_options.high_encode_usage_threshold_percent);
7058 video_stream_encoder_->Stop();
7059}
7060
7061TEST_F(VideoStreamEncoderTest,
7062 HigherCpuAdaptationThresholdsForHardwareEncoder) {
7063 const int kFrameWidth = 1280;
7064 const int kFrameHeight = 720;
7065 CpuOveruseOptions hardware_options;
7066 hardware_options.low_encode_usage_threshold_percent = 150;
7067 hardware_options.high_encode_usage_threshold_percent = 200;
Mirta Dvornicicccc1b572019-01-15 12:42:18 +01007068 fake_encoder_.SetIsHardwareAccelerated(true);
Niels Möller4db138e2018-04-19 09:04:13 +02007069
Henrik Boström381d1092020-05-12 18:49:07 +02007070 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007071 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller4db138e2018-04-19 09:04:13 +02007072 video_source_.IncomingCapturedFrame(
7073 CreateFrame(1, kFrameWidth, kFrameHeight));
7074 WaitForEncodedFrame(1);
7075 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7076 .low_encode_usage_threshold_percent,
7077 hardware_options.low_encode_usage_threshold_percent);
7078 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7079 .high_encode_usage_threshold_percent,
7080 hardware_options.high_encode_usage_threshold_percent);
7081 video_stream_encoder_->Stop();
7082}
7083
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007084TEST_F(VideoStreamEncoderTest,
7085 CpuAdaptationThresholdsUpdatesWhenHardwareAccelerationChange) {
7086 const int kFrameWidth = 1280;
7087 const int kFrameHeight = 720;
7088
7089 const CpuOveruseOptions default_options;
7090 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007091 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Jakob Ivarsson461b1d92021-01-22 16:27:43 +01007092 video_source_.IncomingCapturedFrame(
7093 CreateFrame(1, kFrameWidth, kFrameHeight));
7094 WaitForEncodedFrame(1);
7095 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7096 .low_encode_usage_threshold_percent,
7097 default_options.low_encode_usage_threshold_percent);
7098 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7099 .high_encode_usage_threshold_percent,
7100 default_options.high_encode_usage_threshold_percent);
7101
7102 CpuOveruseOptions hardware_options;
7103 hardware_options.low_encode_usage_threshold_percent = 150;
7104 hardware_options.high_encode_usage_threshold_percent = 200;
7105 fake_encoder_.SetIsHardwareAccelerated(true);
7106
7107 video_source_.IncomingCapturedFrame(
7108 CreateFrame(2, kFrameWidth, kFrameHeight));
7109 WaitForEncodedFrame(2);
7110
7111 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7112 .low_encode_usage_threshold_percent,
7113 hardware_options.low_encode_usage_threshold_percent);
7114 EXPECT_EQ(video_stream_encoder_->overuse_detector_proxy_->GetOptions()
7115 .high_encode_usage_threshold_percent,
7116 hardware_options.high_encode_usage_threshold_percent);
7117
7118 video_stream_encoder_->Stop();
7119}
7120
Niels Möller6bb5ab92019-01-11 11:11:10 +01007121TEST_F(VideoStreamEncoderTest, DropsFramesWhenEncoderOvershoots) {
7122 const int kFrameWidth = 320;
7123 const int kFrameHeight = 240;
7124 const int kFps = 30;
Asa Persson606d3cb2021-10-04 10:07:11 +02007125 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007126 const int kNumFramesInRun = kFps * 5; // Runs of five seconds.
7127
Henrik Boström381d1092020-05-12 18:49:07 +02007128 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007129 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007130
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007131 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007132 max_framerate_ = kFps;
7133
7134 // Insert 3 seconds of video, verify number of drops with normal bitrate.
7135 fake_encoder_.SimulateOvershoot(1.0);
7136 int num_dropped = 0;
7137 for (int i = 0; i < kNumFramesInRun; ++i) {
7138 video_source_.IncomingCapturedFrame(
7139 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7140 // Wait up to two frame durations for a frame to arrive.
7141 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7142 ++num_dropped;
7143 }
7144 timestamp_ms += 1000 / kFps;
7145 }
7146
Erik Språnga8d48ab2019-02-08 14:17:40 +01007147 // Framerate should be measured to be near the expected target rate.
7148 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7149
7150 // Frame drops should be within 5% of expected 0%.
7151 EXPECT_NEAR(num_dropped, 0, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007152
7153 // Make encoder produce frames at double the expected bitrate during 3 seconds
7154 // of video, verify number of drops. Rate needs to be slightly changed in
7155 // order to force the rate to be reconfigured.
Erik Språng7ca375c2019-02-06 16:20:17 +01007156 double overshoot_factor = 2.0;
Erik Språng9d69cbe2020-10-22 17:44:42 +02007157 const RateControlSettings trials =
7158 RateControlSettings::ParseFromFieldTrials();
7159 if (trials.UseEncoderBitrateAdjuster()) {
Erik Språng7ca375c2019-02-06 16:20:17 +01007160 // With bitrate adjuster, when need to overshoot even more to trigger
Erik Språng9d69cbe2020-10-22 17:44:42 +02007161 // frame dropping since the adjuter will try to just lower the target
7162 // bitrate rather than drop frames. If network headroom can be used, it
7163 // doesn't push back as hard so we don't need quite as much overshoot.
7164 // These numbers are unfortunately a bit magical but there's not trivial
7165 // way to algebraically infer them.
Erik Språng73cf80a2021-03-24 11:10:09 +01007166 overshoot_factor = 3.0;
Erik Språng7ca375c2019-02-06 16:20:17 +01007167 }
7168 fake_encoder_.SimulateOvershoot(overshoot_factor);
Henrik Boström381d1092020-05-12 18:49:07 +02007169 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007170 kTargetBitrate + DataRate::KilobitsPerSec(1),
7171 kTargetBitrate + DataRate::KilobitsPerSec(1),
7172 kTargetBitrate + DataRate::KilobitsPerSec(1), 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007173 num_dropped = 0;
7174 for (int i = 0; i < kNumFramesInRun; ++i) {
7175 video_source_.IncomingCapturedFrame(
7176 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7177 // Wait up to two frame durations for a frame to arrive.
7178 if (!TimedWaitForEncodedFrame(timestamp_ms, 2 * 1000 / kFps)) {
7179 ++num_dropped;
7180 }
7181 timestamp_ms += 1000 / kFps;
7182 }
7183
Henrik Boström381d1092020-05-12 18:49:07 +02007184 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007185 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språnga8d48ab2019-02-08 14:17:40 +01007186
7187 // Target framerate should be still be near the expected target, despite
7188 // the frame drops.
7189 EXPECT_NEAR(fake_encoder_.GetLastFramerate(), kFps, 1);
7190
7191 // Frame drops should be within 5% of expected 50%.
7192 EXPECT_NEAR(num_dropped, kNumFramesInRun / 2, 5 * kNumFramesInRun / 100);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007193
7194 video_stream_encoder_->Stop();
7195}
7196
7197TEST_F(VideoStreamEncoderTest, ConfiguresCorrectFrameRate) {
7198 const int kFrameWidth = 320;
7199 const int kFrameHeight = 240;
7200 const int kActualInputFps = 24;
Asa Persson606d3cb2021-10-04 10:07:11 +02007201 const DataRate kTargetBitrate = DataRate::KilobitsPerSec(120);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007202
7203 ASSERT_GT(max_framerate_, kActualInputFps);
7204
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007205 int64_t timestamp_ms = CurrentTimeMs();
Niels Möller6bb5ab92019-01-11 11:11:10 +01007206 max_framerate_ = kActualInputFps;
Henrik Boström381d1092020-05-12 18:49:07 +02007207 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007208 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Niels Möller6bb5ab92019-01-11 11:11:10 +01007209
7210 // Insert 3 seconds of video, with an input fps lower than configured max.
7211 for (int i = 0; i < kActualInputFps * 3; ++i) {
7212 video_source_.IncomingCapturedFrame(
7213 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
7214 // Wait up to two frame durations for a frame to arrive.
7215 WaitForEncodedFrame(timestamp_ms);
7216 timestamp_ms += 1000 / kActualInputFps;
7217 }
7218
7219 EXPECT_NEAR(kActualInputFps, fake_encoder_.GetLastFramerate(), 1);
7220
7221 video_stream_encoder_->Stop();
7222}
7223
Markus Handell9a478b52021-11-18 16:07:01 +01007224TEST_F(VideoStreamEncoderTest, AccumulatesUpdateRectOnDroppedFrames) {
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007225 VideoFrame::UpdateRect rect;
Markus Handell9a478b52021-11-18 16:07:01 +01007226 test::FrameForwarder source;
7227 video_stream_encoder_->SetSource(&source,
7228 DegradationPreference::MAINTAIN_FRAMERATE);
Henrik Boström381d1092020-05-12 18:49:07 +02007229 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007230 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007231
Markus Handell9a478b52021-11-18 16:07:01 +01007232 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(1, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007233 WaitForEncodedFrame(1);
7234 // On the very first frame full update should be forced.
7235 rect = fake_encoder_.GetLastUpdateRect();
7236 EXPECT_EQ(rect.offset_x, 0);
7237 EXPECT_EQ(rect.offset_y, 0);
7238 EXPECT_EQ(rect.height, codec_height_);
7239 EXPECT_EQ(rect.width, codec_width_);
Markus Handell9a478b52021-11-18 16:07:01 +01007240 // Frame with NTP timestamp 2 will be dropped due to outstanding frames
7241 // scheduled for processing during encoder queue processing of frame 2.
7242 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(2, nullptr, 1));
7243 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(3, nullptr, 10));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007244 WaitForEncodedFrame(3);
7245 // Updates to pixels 1 and 10 should be accumulated to one 10x1 rect.
7246 rect = fake_encoder_.GetLastUpdateRect();
7247 EXPECT_EQ(rect.offset_x, 1);
7248 EXPECT_EQ(rect.offset_y, 0);
7249 EXPECT_EQ(rect.width, 10);
7250 EXPECT_EQ(rect.height, 1);
7251
Markus Handell9a478b52021-11-18 16:07:01 +01007252 source.IncomingCapturedFrame(CreateFrameWithUpdatedPixel(4, nullptr, 0));
Ilya Nikolaevskiy71aee3a2019-02-18 13:01:26 +01007253 WaitForEncodedFrame(4);
7254 // Previous frame was encoded, so no accumulation should happen.
7255 rect = fake_encoder_.GetLastUpdateRect();
7256 EXPECT_EQ(rect.offset_x, 0);
7257 EXPECT_EQ(rect.offset_y, 0);
7258 EXPECT_EQ(rect.width, 1);
7259 EXPECT_EQ(rect.height, 1);
7260
7261 video_stream_encoder_->Stop();
7262}
7263
Erik Språngd7329ca2019-02-21 21:19:53 +01007264TEST_F(VideoStreamEncoderTest, SetsFrameTypes) {
Henrik Boström381d1092020-05-12 18:49:07 +02007265 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007266 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007267
7268 // First frame is always keyframe.
7269 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7270 WaitForEncodedFrame(1);
Niels Möller8f7ce222019-03-21 15:43:58 +01007271 EXPECT_THAT(
7272 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007273 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007274
7275 // Insert delta frame.
7276 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7277 WaitForEncodedFrame(2);
Niels Möller8f7ce222019-03-21 15:43:58 +01007278 EXPECT_THAT(
7279 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007280 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007281
7282 // Request next frame be a key-frame.
7283 video_stream_encoder_->SendKeyFrame();
7284 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7285 WaitForEncodedFrame(3);
Niels Möller8f7ce222019-03-21 15:43:58 +01007286 EXPECT_THAT(
7287 fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007288 ::testing::ElementsAre(VideoFrameType{VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007289
7290 video_stream_encoder_->Stop();
7291}
7292
7293TEST_F(VideoStreamEncoderTest, SetsFrameTypesSimulcast) {
7294 // Setup simulcast with three streams.
7295 ResetEncoder("VP8", 3, 1, 1, false);
Henrik Boström381d1092020-05-12 18:49:07 +02007296 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007297 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7298 0, 0, 0);
Erik Språngd7329ca2019-02-21 21:19:53 +01007299 // Wait for all three layers before triggering event.
7300 sink_.SetNumExpectedLayers(3);
7301
7302 // First frame is always keyframe.
7303 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7304 WaitForEncodedFrame(1);
7305 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007306 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7307 VideoFrameType::kVideoFrameKey,
7308 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007309
7310 // Insert delta frame.
7311 video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr));
7312 WaitForEncodedFrame(2);
7313 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007314 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7315 VideoFrameType::kVideoFrameDelta,
7316 VideoFrameType::kVideoFrameDelta}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007317
7318 // Request next frame be a key-frame.
7319 // Only first stream is configured to produce key-frame.
7320 video_stream_encoder_->SendKeyFrame();
7321 video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
7322 WaitForEncodedFrame(3);
Sergey Silkine62a08a2019-05-13 13:45:39 +02007323
7324 // TODO(webrtc:10615): Map keyframe request to spatial layer. Currently
7325 // keyframe request on any layer triggers keyframe on all layers.
Erik Språngd7329ca2019-02-21 21:19:53 +01007326 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
Mirko Bonadei6a489f22019-04-09 15:11:12 +02007327 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
Sergey Silkine62a08a2019-05-13 13:45:39 +02007328 VideoFrameType::kVideoFrameKey,
7329 VideoFrameType::kVideoFrameKey}));
Erik Språngd7329ca2019-02-21 21:19:53 +01007330
7331 video_stream_encoder_->Stop();
7332}
7333
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007334TEST_F(VideoStreamEncoderTest, DoesNotRewriteH264BitstreamWithOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007335 // SPS contains VUI with restrictions on the maximum number of reordered
7336 // pictures, there is no need to rewrite the bitstream to enable faster
7337 // decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007338 ResetEncoder("H264", 1, 1, 1, false);
7339
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007340 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007341 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007342 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007343
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007344 fake_encoder_.SetEncodedImageData(
Asa Persson606d3cb2021-10-04 10:07:11 +02007345 EncodedImageBuffer::Create(kOptimalSps, sizeof(kOptimalSps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007346
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007347 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7348 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007349
7350 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007351 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007352
7353 video_stream_encoder_->Stop();
7354}
7355
7356TEST_F(VideoStreamEncoderTest, RewritesH264BitstreamWithNonOptimalSps) {
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007357 // SPS does not contain VUI, the bitstream is will be rewritten with added
7358 // VUI with restrictions on the maximum number of reordered pictures to
7359 // enable faster decoding.
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007360 uint8_t original_sps[] = {0, 0, 0, 1, H264::NaluType::kSps,
7361 0x00, 0x00, 0x03, 0x03, 0xF4,
7362 0x05, 0x03, 0xC7, 0xC0};
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007363 ResetEncoder("H264", 1, 1, 1, false);
7364
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007365 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007366 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007367 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007368
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007369 fake_encoder_.SetEncodedImageData(
7370 EncodedImageBuffer::Create(original_sps, sizeof(original_sps)));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007371
Mirta Dvornicic97910da2020-07-14 15:29:23 +02007372 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7373 WaitForEncodedFrame(1);
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007374
7375 EXPECT_THAT(sink_.GetLastEncodedImageData(),
Asa Persson606d3cb2021-10-04 10:07:11 +02007376 testing::ElementsAreArray(kOptimalSps));
Mirta Dvornicic28f0eb22019-05-28 16:30:16 +02007377
7378 video_stream_encoder_->Stop();
7379}
7380
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007381TEST_F(VideoStreamEncoderTest, CopiesVideoFrameMetadataAfterDownscale) {
7382 const int kFrameWidth = 1280;
7383 const int kFrameHeight = 720;
Asa Persson606d3cb2021-10-04 10:07:11 +02007384 const DataRate kTargetBitrate =
7385 DataRate::KilobitsPerSec(300); // Too low for HD resolution.
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007386
Henrik Boström381d1092020-05-12 18:49:07 +02007387 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007388 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007389 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7390
7391 // Insert a first video frame. It should be dropped because of downscale in
7392 // resolution.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007393 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007394 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7395 frame.set_rotation(kVideoRotation_270);
7396 video_source_.IncomingCapturedFrame(frame);
7397
7398 ExpectDroppedFrame();
7399
7400 // Second frame is downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007401 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007402 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7403 frame.set_rotation(kVideoRotation_90);
7404 video_source_.IncomingCapturedFrame(frame);
7405
7406 WaitForEncodedFrame(timestamp_ms);
7407 sink_.CheckLastFrameRotationMatches(kVideoRotation_90);
7408
7409 // Insert another frame, also downscaled.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007410 timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiyba96e2f2019-06-04 15:15:14 +02007411 frame = CreateFrame(timestamp_ms, kFrameWidth / 2, kFrameHeight / 2);
7412 frame.set_rotation(kVideoRotation_180);
7413 video_source_.IncomingCapturedFrame(frame);
7414
7415 WaitForEncodedFrame(timestamp_ms);
7416 sink_.CheckLastFrameRotationMatches(kVideoRotation_180);
7417
7418 video_stream_encoder_->Stop();
7419}
7420
Erik Språng5056af02019-09-02 15:53:11 +02007421TEST_F(VideoStreamEncoderTest, BandwidthAllocationLowerBound) {
7422 const int kFrameWidth = 320;
7423 const int kFrameHeight = 180;
7424
7425 // Initial rate.
Henrik Boström381d1092020-05-12 18:49:07 +02007426 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007427 /*target_bitrate=*/DataRate::KilobitsPerSec(300),
7428 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(300),
7429 /*link_allocation=*/DataRate::KilobitsPerSec(300),
Erik Språng5056af02019-09-02 15:53:11 +02007430 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007431 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007432 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007433
7434 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007435 int64_t timestamp_ms = CurrentTimeMs();
Erik Språng5056af02019-09-02 15:53:11 +02007436 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7437 frame.set_rotation(kVideoRotation_270);
7438 video_source_.IncomingCapturedFrame(frame);
7439 WaitForEncodedFrame(timestamp_ms);
7440
7441 // Set a target rate below the minimum allowed by the codec settings.
Asa Persson606d3cb2021-10-04 10:07:11 +02007442 VideoCodec codec_config = fake_encoder_.config();
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007443 DataRate min_rate = DataRate::KilobitsPerSec(codec_config.minBitrate);
7444 DataRate target_rate = min_rate - DataRate::KilobitsPerSec(1);
Henrik Boström381d1092020-05-12 18:49:07 +02007445 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Erik Språng5056af02019-09-02 15:53:11 +02007446 /*target_bitrate=*/target_rate,
Florent Castellia8336d32019-09-09 13:36:55 +02007447 /*stable_target_bitrate=*/target_rate,
Erik Språng5056af02019-09-02 15:53:11 +02007448 /*link_allocation=*/target_rate,
7449 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007450 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007451 /*cwnd_reduce_ratio=*/0);
Erik Språng5056af02019-09-02 15:53:11 +02007452 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7453
7454 // Target bitrate and bandwidth allocation should both be capped at min_rate.
7455 auto rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7456 ASSERT_TRUE(rate_settings.has_value());
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007457 DataRate allocation_sum =
7458 DataRate::BitsPerSec(rate_settings->bitrate.get_sum_bps());
Erik Språng5056af02019-09-02 15:53:11 +02007459 EXPECT_EQ(min_rate, allocation_sum);
7460 EXPECT_EQ(rate_settings->bandwidth_allocation, min_rate);
7461
7462 video_stream_encoder_->Stop();
7463}
7464
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007465TEST_F(VideoStreamEncoderTest, EncoderRatesPropagatedOnReconfigure) {
Henrik Boström381d1092020-05-12 18:49:07 +02007466 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007467 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007468 // Capture a frame and wait for it to synchronize with the encoder thread.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007469 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007470 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7471 WaitForEncodedFrame(1);
7472
7473 auto prev_rate_settings = fake_encoder_.GetAndResetLastRateControlSettings();
7474 ASSERT_TRUE(prev_rate_settings.has_value());
7475 EXPECT_EQ(static_cast<int>(prev_rate_settings->framerate_fps),
7476 kDefaultFramerate);
7477
7478 // Send 1s of video to ensure the framerate is stable at kDefaultFramerate.
7479 for (int i = 0; i < 2 * kDefaultFramerate; i++) {
7480 timestamp_ms += 1000 / kDefaultFramerate;
7481 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7482 WaitForEncodedFrame(timestamp_ms);
7483 }
7484 EXPECT_EQ(static_cast<int>(fake_encoder_.GetLastFramerate()),
7485 kDefaultFramerate);
7486 // Capture larger frame to trigger a reconfigure.
7487 codec_height_ *= 2;
7488 codec_width_ *= 2;
7489 timestamp_ms += 1000 / kDefaultFramerate;
7490 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, nullptr));
7491 WaitForEncodedFrame(timestamp_ms);
7492
7493 EXPECT_EQ(2, sink_.number_of_reconfigurations());
7494 auto current_rate_settings =
7495 fake_encoder_.GetAndResetLastRateControlSettings();
7496 // Ensure we have actually reconfigured twice
7497 // The rate settings should have been set again even though
7498 // they haven't changed.
7499 ASSERT_TRUE(current_rate_settings.has_value());
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007500 EXPECT_EQ(prev_rate_settings, current_rate_settings);
Evan Shrubsolee32ae4f2019-09-25 12:50:23 +02007501
7502 video_stream_encoder_->Stop();
7503}
7504
philipeld9cc8c02019-09-16 14:53:40 +02007505struct MockEncoderSwitchRequestCallback : public EncoderSwitchRequestCallback {
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007506 MOCK_METHOD(void, RequestEncoderFallback, (), (override));
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007507 MOCK_METHOD(void,
7508 RequestEncoderSwitch,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007509 (const webrtc::SdpVideoFormat& format,
7510 bool allow_default_fallback),
Danil Chapovalov91fdc602020-05-14 19:17:51 +02007511 (override));
philipeld9cc8c02019-09-16 14:53:40 +02007512};
7513
philipel9b058032020-02-10 11:30:00 +01007514TEST_F(VideoStreamEncoderTest, EncoderSelectorCurrentEncoderIsSignaled) {
7515 constexpr int kDontCare = 100;
7516 StrictMock<MockEncoderSelector> encoder_selector;
7517 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7518 &fake_encoder_, &encoder_selector);
7519 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7520
7521 // Reset encoder for new configuration to take effect.
7522 ConfigureEncoder(video_encoder_config_.Copy());
7523
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007524 EXPECT_CALL(encoder_selector, OnCurrentEncoder);
philipel9b058032020-02-10 11:30:00 +01007525
7526 video_source_.IncomingCapturedFrame(
7527 CreateFrame(kDontCare, kDontCare, kDontCare));
Markus Handell28c71802021-11-08 10:11:55 +01007528 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007529 video_stream_encoder_->Stop();
7530
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007531 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007532 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007533 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7534 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007535 video_stream_encoder_.reset();
7536}
7537
7538TEST_F(VideoStreamEncoderTest, EncoderSelectorBitrateSwitch) {
7539 constexpr int kDontCare = 100;
7540
7541 NiceMock<MockEncoderSelector> encoder_selector;
7542 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7543 video_send_config_.encoder_settings.encoder_switch_request_callback =
7544 &switch_callback;
7545 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7546 &fake_encoder_, &encoder_selector);
7547 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7548
7549 // Reset encoder for new configuration to take effect.
7550 ConfigureEncoder(video_encoder_config_.Copy());
7551
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007552 ON_CALL(encoder_selector, OnAvailableBitrate)
philipel9b058032020-02-10 11:30:00 +01007553 .WillByDefault(Return(SdpVideoFormat("AV1")));
7554 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007555 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7556 /*allow_default_fallback=*/false));
philipel9b058032020-02-10 11:30:00 +01007557
Henrik Boström381d1092020-05-12 18:49:07 +02007558 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007559 /*target_bitrate=*/DataRate::KilobitsPerSec(50),
7560 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(kDontCare),
7561 /*link_allocation=*/DataRate::KilobitsPerSec(kDontCare),
philipel9b058032020-02-10 11:30:00 +01007562 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007563 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007564 /*cwnd_reduce_ratio=*/0);
Markus Handell28c71802021-11-08 10:11:55 +01007565 AdvanceTime(TimeDelta::Zero());
philipel9b058032020-02-10 11:30:00 +01007566
7567 video_stream_encoder_->Stop();
7568}
7569
philipel6daa3042022-04-11 10:48:28 +02007570TEST_F(VideoStreamEncoderTest, EncoderSelectorResolutionSwitch) {
7571 NiceMock<MockEncoderSelector> encoder_selector;
7572 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7573 video_send_config_.encoder_settings.encoder_switch_request_callback =
7574 &switch_callback;
7575 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7576 &fake_encoder_, &encoder_selector);
7577 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7578
7579 // Reset encoder for new configuration to take effect.
7580 ConfigureEncoder(video_encoder_config_.Copy());
7581
7582 EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(640, 480)))
7583 .WillOnce(Return(absl::nullopt));
7584 EXPECT_CALL(encoder_selector, OnResolutionChange(RenderResolution(320, 240)))
7585 .WillOnce(Return(SdpVideoFormat("AV1")));
7586 EXPECT_CALL(switch_callback,
7587 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV1"),
7588 /*allow_default_fallback=*/false));
7589
7590 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7591 /*target_bitrate=*/DataRate::KilobitsPerSec(800),
7592 /*stable_target_bitrate=*/DataRate::KilobitsPerSec(1000),
7593 /*link_allocation=*/DataRate::KilobitsPerSec(1000),
7594 /*fraction_lost=*/0,
7595 /*round_trip_time_ms=*/0,
7596 /*cwnd_reduce_ratio=*/0);
7597
7598 video_source_.IncomingCapturedFrame(CreateFrame(1, 640, 480));
7599 video_source_.IncomingCapturedFrame(CreateFrame(2, 640, 480));
7600 video_source_.IncomingCapturedFrame(CreateFrame(3, 320, 240));
7601
7602 AdvanceTime(TimeDelta::Zero());
7603
7604 video_stream_encoder_->Stop();
7605}
7606
philipel9b058032020-02-10 11:30:00 +01007607TEST_F(VideoStreamEncoderTest, EncoderSelectorBrokenEncoderSwitch) {
7608 constexpr int kSufficientBitrateToNotDrop = 1000;
7609 constexpr int kDontCare = 100;
7610
7611 NiceMock<MockVideoEncoder> video_encoder;
7612 NiceMock<MockEncoderSelector> encoder_selector;
7613 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7614 video_send_config_.encoder_settings.encoder_switch_request_callback =
7615 &switch_callback;
7616 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7617 &video_encoder, &encoder_selector);
7618 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7619
7620 // Reset encoder for new configuration to take effect.
7621 ConfigureEncoder(video_encoder_config_.Copy());
7622
7623 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7624 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7625 // not fail.
Henrik Boström381d1092020-05-12 18:49:07 +02007626 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007627 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7628 /*stable_target_bitrate=*/
7629 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7630 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
philipel9b058032020-02-10 11:30:00 +01007631 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007632 /*round_trip_time_ms=*/0,
philipel9b058032020-02-10 11:30:00 +01007633 /*cwnd_reduce_ratio=*/0);
7634
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007635 ON_CALL(video_encoder, Encode)
philipel9b058032020-02-10 11:30:00 +01007636 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007637 ON_CALL(encoder_selector, OnEncoderBroken)
philipel9b058032020-02-10 11:30:00 +01007638 .WillByDefault(Return(SdpVideoFormat("AV2")));
7639
7640 rtc::Event encode_attempted;
7641 EXPECT_CALL(switch_callback,
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007642 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7643 /*allow_default_fallback=*/true))
7644 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
philipel9b058032020-02-10 11:30:00 +01007645
7646 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7647 encode_attempted.Wait(3000);
7648
Markus Handell28c71802021-11-08 10:11:55 +01007649 AdvanceTime(TimeDelta::Zero());
Tomas Gunnarssond41c2a62020-09-21 15:56:42 +02007650
philipel9b058032020-02-10 11:30:00 +01007651 video_stream_encoder_->Stop();
7652
Sergey Silkine1cd3ad2022-01-21 11:35:04 +01007653 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7654 // to it's factory, so in order for the encoder instance in the
7655 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7656 // reset the `video_stream_encoder_` here.
7657 video_stream_encoder_.reset();
7658}
7659
7660TEST_F(VideoStreamEncoderTest, SwitchEncoderOnInitFailureWithEncoderSelector) {
7661 NiceMock<MockVideoEncoder> video_encoder;
7662 NiceMock<MockEncoderSelector> encoder_selector;
7663 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7664 video_send_config_.encoder_settings.encoder_switch_request_callback =
7665 &switch_callback;
7666 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7667 &video_encoder, &encoder_selector);
7668 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7669
7670 // Reset encoder for new configuration to take effect.
7671 ConfigureEncoder(video_encoder_config_.Copy());
7672
7673 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7674 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7675 /*round_trip_time_ms=*/0,
7676 /*cwnd_reduce_ratio=*/0);
7677 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7678
7679 ON_CALL(video_encoder, InitEncode(_, _))
7680 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7681 ON_CALL(encoder_selector, OnEncoderBroken)
7682 .WillByDefault(Return(SdpVideoFormat("AV2")));
7683
7684 rtc::Event encode_attempted;
7685 EXPECT_CALL(switch_callback,
7686 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7687 /*allow_default_fallback=*/true))
7688 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7689
7690 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7691 encode_attempted.Wait(3000);
7692
7693 AdvanceTime(TimeDelta::Zero());
7694
7695 video_stream_encoder_->Stop();
7696
7697 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7698 // to it's factory, so in order for the encoder instance in the
7699 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7700 // reset the `video_stream_encoder_` here.
7701 video_stream_encoder_.reset();
7702}
7703
7704TEST_F(VideoStreamEncoderTest,
7705 SwitchEncoderOnInitFailureWithoutEncoderSelector) {
7706 NiceMock<MockVideoEncoder> video_encoder;
7707 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7708 video_send_config_.encoder_settings.encoder_switch_request_callback =
7709 &switch_callback;
7710 auto encoder_factory = std::make_unique<test::VideoEncoderProxyFactory>(
7711 &video_encoder, /*encoder_selector=*/nullptr);
7712 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7713
7714 // Reset encoder for new configuration to take effect.
7715 ConfigureEncoder(video_encoder_config_.Copy());
7716
7717 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7718 kTargetBitrate, kTargetBitrate, kTargetBitrate, /*fraction_lost=*/0,
7719 /*round_trip_time_ms=*/0,
7720 /*cwnd_reduce_ratio=*/0);
7721 ASSERT_EQ(0, sink_.number_of_reconfigurations());
7722
7723 ON_CALL(video_encoder, InitEncode(_, _))
7724 .WillByDefault(Return(WEBRTC_VIDEO_CODEC_ENCODER_FAILURE));
7725
7726 rtc::Event encode_attempted;
7727 EXPECT_CALL(switch_callback,
7728 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "VP8"),
7729 /*allow_default_fallback=*/true))
7730 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7731
7732 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
7733 encode_attempted.Wait(3000);
7734
7735 AdvanceTime(TimeDelta::Zero());
7736
7737 video_stream_encoder_->Stop();
7738
7739 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
philipel9b058032020-02-10 11:30:00 +01007740 // to it's factory, so in order for the encoder instance in the
Artem Titovab30d722021-07-27 16:22:11 +02007741 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7742 // reset the `video_stream_encoder_` here.
philipel9b058032020-02-10 11:30:00 +01007743 video_stream_encoder_.reset();
7744}
7745
Byoungchan Lee13fe3672022-04-06 10:44:42 +09007746TEST_F(VideoStreamEncoderTest, NullEncoderReturnSwitch) {
7747 // As a variant of EncoderSelectorBrokenEncoderSwitch, when a null
7748 // VideoEncoder is passed in encoder_factory, it checks whether
7749 // Codec Switch occurs without a crash.
7750 constexpr int kSufficientBitrateToNotDrop = 1000;
7751 constexpr int kDontCare = 100;
7752
7753 NiceMock<MockEncoderSelector> encoder_selector;
7754 StrictMock<MockEncoderSwitchRequestCallback> switch_callback;
7755 video_send_config_.encoder_settings.encoder_switch_request_callback =
7756 &switch_callback;
7757 auto encoder_factory =
7758 std::make_unique<test::VideoEncoderNullableProxyFactory>(
7759 /*encoder=*/nullptr, &encoder_selector);
7760 video_send_config_.encoder_settings.encoder_factory = encoder_factory.get();
7761
7762 // Reset encoder for new configuration to take effect.
7763 ConfigureEncoder(video_encoder_config_.Copy());
7764 // The VideoStreamEncoder needs some bitrate before it can start encoding,
7765 // setting some bitrate so that subsequent calls to WaitForEncodedFrame does
7766 // not fail.
7767 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
7768 /*target_bitrate=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7769 /*stable_target_bitrate=*/
7770 DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7771 /*link_allocation=*/DataRate::KilobitsPerSec(kSufficientBitrateToNotDrop),
7772 /*fraction_lost=*/0,
7773 /*round_trip_time_ms=*/0,
7774 /*cwnd_reduce_ratio=*/0);
7775 ON_CALL(encoder_selector, OnEncoderBroken)
7776 .WillByDefault(Return(SdpVideoFormat("AV2")));
7777 rtc::Event encode_attempted;
7778 EXPECT_CALL(switch_callback,
7779 RequestEncoderSwitch(Field(&SdpVideoFormat::name, "AV2"),
7780 /*allow_default_fallback=*/_))
7781 .WillOnce([&encode_attempted]() { encode_attempted.Set(); });
7782
7783 video_source_.IncomingCapturedFrame(CreateFrame(1, kDontCare, kDontCare));
7784 encode_attempted.Wait(3000);
7785
7786 AdvanceTime(TimeDelta::Zero());
7787
7788 video_stream_encoder_->Stop();
7789
7790 // The encoders produced by the VideoEncoderProxyFactory have a pointer back
7791 // to it's factory, so in order for the encoder instance in the
7792 // `video_stream_encoder_` to be destroyed before the `encoder_factory` we
7793 // reset the `video_stream_encoder_` here.
7794 video_stream_encoder_.reset();
7795}
7796
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007797TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007798 AllocationPropagatedToEncoderWhenTargetRateChanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007799 const int kFrameWidth = 320;
7800 const int kFrameHeight = 180;
7801
7802 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007803 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007804 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007805 /*target_bitrate=*/rate,
7806 /*stable_target_bitrate=*/rate,
7807 /*link_allocation=*/rate,
7808 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007809 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007810 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007811
7812 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007813 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007814 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7815 frame.set_rotation(kVideoRotation_270);
7816 video_source_.IncomingCapturedFrame(frame);
7817 WaitForEncodedFrame(timestamp_ms);
7818 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7819
7820 // Change of target bitrate propagates to the encoder.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007821 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007822 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007823 /*target_bitrate=*/new_stable_rate,
7824 /*stable_target_bitrate=*/new_stable_rate,
7825 /*link_allocation=*/rate,
7826 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007827 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007828 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007829 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7830 EXPECT_EQ(2, fake_encoder_.GetNumSetRates());
7831 video_stream_encoder_->Stop();
7832}
7833
7834TEST_F(VideoStreamEncoderTest,
Rasmus Brandt5cad55b2019-12-19 09:47:11 +01007835 AllocationNotPropagatedToEncoderWhenTargetRateUnchanged) {
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007836 const int kFrameWidth = 320;
7837 const int kFrameHeight = 180;
7838
7839 // Set initial rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007840 auto rate = DataRate::KilobitsPerSec(100);
Henrik Boström381d1092020-05-12 18:49:07 +02007841 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007842 /*target_bitrate=*/rate,
7843 /*stable_target_bitrate=*/rate,
7844 /*link_allocation=*/rate,
7845 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007846 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007847 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007848
7849 // Insert a first video frame so that encoder gets configured.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007850 int64_t timestamp_ms = CurrentTimeMs();
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007851 VideoFrame frame = CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight);
7852 frame.set_rotation(kVideoRotation_270);
7853 video_source_.IncomingCapturedFrame(frame);
7854 WaitForEncodedFrame(timestamp_ms);
7855 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7856
7857 // Set a higher target rate without changing the link_allocation. Should not
7858 // reset encoder's rate.
Danil Chapovalovcad3e0e2020-02-17 18:46:07 +01007859 auto new_stable_rate = rate - DataRate::KilobitsPerSec(5);
Henrik Boström381d1092020-05-12 18:49:07 +02007860 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007861 /*target_bitrate=*/rate,
7862 /*stable_target_bitrate=*/new_stable_rate,
7863 /*link_allocation=*/rate,
7864 /*fraction_lost=*/0,
Åsa Persson4d4f62f2021-09-12 16:14:48 +02007865 /*round_trip_time_ms=*/0,
Ying Wang9b881ab2020-02-07 14:29:32 +01007866 /*cwnd_reduce_ratio=*/0);
Evan Shrubsole7c079f62019-09-26 09:55:03 +02007867 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
7868 EXPECT_EQ(1, fake_encoder_.GetNumSetRates());
7869 video_stream_encoder_->Stop();
7870}
7871
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007872TEST_F(VideoStreamEncoderTest, AutomaticAnimationDetection) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01007873 test::ScopedKeyValueConfig field_trials(
7874 field_trials_,
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007875 "WebRTC-AutomaticAnimationDetectionScreenshare/"
7876 "enabled:true,min_fps:20,min_duration_ms:1000,min_area_ratio:0.8/");
7877 const int kFramerateFps = 30;
7878 const int kWidth = 1920;
7879 const int kHeight = 1080;
7880 const int kNumFrames = 2 * kFramerateFps; // >1 seconds of frames.
7881 // Works on screenshare mode.
7882 ResetEncoder("VP8", 1, 1, 1, /*screenshare*/ true);
7883 // We rely on the automatic resolution adaptation, but we handle framerate
7884 // adaptation manually by mocking the stats proxy.
7885 video_source_.set_adaptation_enabled(true);
7886
7887 // BALANCED degradation preference is required for this feature.
Henrik Boström381d1092020-05-12 18:49:07 +02007888 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007889 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007890 video_stream_encoder_->SetSource(&video_source_,
7891 webrtc::DegradationPreference::BALANCED);
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007892 EXPECT_THAT(video_source_.sink_wants(), UnlimitedSinkWants());
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007893
7894 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7895 frame.set_update_rect(VideoFrame::UpdateRect{0, 0, kWidth, kHeight});
7896
7897 // Pass enough frames with the full update to trigger animation detection.
7898 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007899 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007900 frame.set_ntp_time_ms(timestamp_ms);
7901 frame.set_timestamp_us(timestamp_ms * 1000);
7902 video_source_.IncomingCapturedFrame(frame);
7903 WaitForEncodedFrame(timestamp_ms);
7904 }
7905
7906 // Resolution should be limited.
7907 rtc::VideoSinkWants expected;
7908 expected.max_framerate_fps = kFramerateFps;
7909 expected.max_pixel_count = 1280 * 720 + 1;
Evan Shrubsole5fd40602020-05-25 16:19:54 +02007910 EXPECT_THAT(video_source_.sink_wants(), FpsEqResolutionLt(expected));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007911
7912 // Pass one frame with no known update.
7913 // Resolution cap should be removed immediately.
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007914 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007915 frame.set_ntp_time_ms(timestamp_ms);
7916 frame.set_timestamp_us(timestamp_ms * 1000);
7917 frame.clear_update_rect();
7918
7919 video_source_.IncomingCapturedFrame(frame);
7920 WaitForEncodedFrame(timestamp_ms);
7921
7922 // Resolution should be unlimited now.
Evan Shrubsole5cd7eb82020-05-25 14:08:39 +02007923 EXPECT_THAT(video_source_.sink_wants(),
7924 FpsMatchesResolutionMax(Eq(kFramerateFps)));
Ilya Nikolaevskiy648b9d72019-12-03 16:54:17 +01007925
7926 video_stream_encoder_->Stop();
7927}
7928
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007929TEST_F(VideoStreamEncoderTest, ConfiguresVp9SvcAtOddResolutions) {
7930 const int kWidth = 720; // 540p adapted down.
7931 const int kHeight = 405;
7932 const int kNumFrames = 3;
7933 // Works on screenshare mode.
7934 ResetEncoder("VP9", /*num_streams=*/1, /*num_temporal_layers=*/1,
7935 /*num_spatial_layers=*/2, /*screenshare=*/true);
7936
7937 video_source_.set_adaptation_enabled(true);
7938
7939 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007940 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007941
7942 VideoFrame frame = CreateFrame(1, kWidth, kHeight);
7943
7944 // Pass enough frames with the full update to trigger animation detection.
7945 for (int i = 0; i < kNumFrames; ++i) {
Tomas Gunnarsson612445e2020-09-21 14:31:23 +02007946 int64_t timestamp_ms = CurrentTimeMs();
Ilya Nikolaevskiy09eb6e22020-06-05 12:36:32 +02007947 frame.set_ntp_time_ms(timestamp_ms);
7948 frame.set_timestamp_us(timestamp_ms * 1000);
7949 video_source_.IncomingCapturedFrame(frame);
7950 WaitForEncodedFrame(timestamp_ms);
7951 }
7952
7953 video_stream_encoder_->Stop();
7954}
7955
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007956TEST_F(VideoStreamEncoderTest, EncoderResetAccordingToParameterChange) {
7957 const float downscale_factors[] = {4.0, 2.0, 1.0};
7958 const int number_layers =
7959 sizeof(downscale_factors) / sizeof(downscale_factors[0]);
7960 VideoEncoderConfig config;
7961 test::FillEncoderConfiguration(kVideoCodecVP8, number_layers, &config);
7962 for (int i = 0; i < number_layers; ++i) {
7963 config.simulcast_layers[i].scale_resolution_down_by = downscale_factors[i];
7964 config.simulcast_layers[i].active = true;
7965 }
7966 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02007967 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007968 "VP8", /*max qp*/ 56, /*screencast*/ false,
7969 /*screenshare enabled*/ false);
7970 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02007971 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
7972 0, 0, 0);
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007973
7974 // First initialization.
7975 // Encoder should be initialized. Next frame should be key frame.
7976 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7977 sink_.SetNumExpectedLayers(number_layers);
7978 int64_t timestamp_ms = kFrameIntervalMs;
7979 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7980 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007981 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007982 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7983 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
7984 VideoFrameType::kVideoFrameKey,
7985 VideoFrameType::kVideoFrameKey}));
7986
7987 // Disable top layer.
7988 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
7989 config.simulcast_layers[number_layers - 1].active = false;
7990 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
7991 sink_.SetNumExpectedLayers(number_layers - 1);
7992 timestamp_ms += kFrameIntervalMs;
7993 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
7994 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02007995 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07007996 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
7997 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
7998 VideoFrameType::kVideoFrameDelta,
7999 VideoFrameType::kVideoFrameDelta}));
8000
8001 // Re-enable top layer.
8002 // Encoder should be re-initialized. Next frame should be key frame.
8003 config.simulcast_layers[number_layers - 1].active = true;
8004 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8005 sink_.SetNumExpectedLayers(number_layers);
8006 timestamp_ms += kFrameIntervalMs;
8007 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8008 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008009 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008010 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8011 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
8012 VideoFrameType::kVideoFrameKey,
8013 VideoFrameType::kVideoFrameKey}));
8014
8015 // Top layer max rate change.
8016 // Encoder shouldn't be re-initialized. Next frame should be delta frame.
8017 config.simulcast_layers[number_layers - 1].max_bitrate_bps -= 100;
8018 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8019 sink_.SetNumExpectedLayers(number_layers);
8020 timestamp_ms += kFrameIntervalMs;
8021 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8022 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008023 EXPECT_EQ(2, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008024 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8025 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameDelta,
8026 VideoFrameType::kVideoFrameDelta,
8027 VideoFrameType::kVideoFrameDelta}));
8028
8029 // Top layer resolution change.
8030 // Encoder should be re-initialized. Next frame should be key frame.
8031 config.simulcast_layers[number_layers - 1].scale_resolution_down_by += 0.1;
8032 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8033 sink_.SetNumExpectedLayers(number_layers);
8034 timestamp_ms += kFrameIntervalMs;
8035 video_source_.IncomingCapturedFrame(CreateFrame(timestamp_ms, 1280, 720));
8036 WaitForEncodedFrame(timestamp_ms);
Asa Persson606d3cb2021-10-04 10:07:11 +02008037 EXPECT_EQ(3, fake_encoder_.GetNumInitializations());
Yun Zhang1e4d4fd2020-09-30 00:22:46 -07008038 EXPECT_THAT(fake_encoder_.LastFrameTypes(),
8039 ::testing::ElementsAreArray({VideoFrameType::kVideoFrameKey,
8040 VideoFrameType::kVideoFrameKey,
8041 VideoFrameType::kVideoFrameKey}));
8042 video_stream_encoder_->Stop();
8043}
8044
Henrik Boström1124ed12021-02-25 10:30:39 +01008045TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSinglecast) {
8046 const int kFrameWidth = 1280;
8047 const int kFrameHeight = 720;
8048
8049 SetUp();
8050 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008051 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01008052
8053 // Capturing a frame should reconfigure the encoder and expose the encoder
8054 // resolution, which is the same as the input frame.
8055 int64_t timestamp_ms = kFrameIntervalMs;
8056 video_source_.IncomingCapturedFrame(
8057 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8058 WaitForEncodedFrame(timestamp_ms);
8059 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8060 EXPECT_THAT(video_source_.sink_wants().resolutions,
8061 ::testing::ElementsAreArray(
8062 {rtc::VideoSinkWants::FrameSize(kFrameWidth, kFrameHeight)}));
8063
8064 video_stream_encoder_->Stop();
8065}
8066
8067TEST_F(VideoStreamEncoderTest, EncoderResolutionsExposedInSimulcast) {
8068 // Pick downscale factors such that we never encode at full resolution - this
8069 // is an interesting use case. The frame resolution influences the encoder
Artem Titovab30d722021-07-27 16:22:11 +02008070 // resolutions, but if no layer has `scale_resolution_down_by` == 1 then the
Henrik Boström1124ed12021-02-25 10:30:39 +01008071 // encoder should not ask for the frame resolution. This allows video frames
8072 // to have the appearence of one resolution but optimize its internal buffers
8073 // for what is actually encoded.
8074 const size_t kNumSimulcastLayers = 3u;
8075 const float kDownscaleFactors[] = {8.0, 4.0, 2.0};
8076 const int kFrameWidth = 1280;
8077 const int kFrameHeight = 720;
8078 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8079 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8080 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8081 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8082 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8083 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8084
8085 VideoEncoderConfig config;
8086 test::FillEncoderConfiguration(kVideoCodecVP8, kNumSimulcastLayers, &config);
8087 for (size_t i = 0; i < kNumSimulcastLayers; ++i) {
8088 config.simulcast_layers[i].scale_resolution_down_by = kDownscaleFactors[i];
8089 config.simulcast_layers[i].active = true;
8090 }
8091 config.video_stream_factory =
Tomas Gunnarssonc1d58912021-04-22 19:21:43 +02008092 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
Henrik Boström1124ed12021-02-25 10:30:39 +01008093 "VP8", /*max qp*/ 56, /*screencast*/ false,
8094 /*screenshare enabled*/ false);
8095 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008096 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8097 0, 0, 0);
Henrik Boström1124ed12021-02-25 10:30:39 +01008098
8099 // Capture a frame with all layers active.
8100 int64_t timestamp_ms = kFrameIntervalMs;
8101 sink_.SetNumExpectedLayers(kNumSimulcastLayers);
8102 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8103 video_source_.IncomingCapturedFrame(
8104 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8105 WaitForEncodedFrame(timestamp_ms);
8106 // Expect encoded resolutions to match the expected simulcast layers.
8107 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8108 EXPECT_THAT(
8109 video_source_.sink_wants().resolutions,
8110 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size, kLayer2Size}));
8111
8112 // Capture a frame with one of the layers inactive.
8113 timestamp_ms += kFrameIntervalMs;
8114 config.simulcast_layers[2].active = false;
8115 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 1);
8116 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8117 video_source_.IncomingCapturedFrame(
8118 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8119 WaitForEncodedFrame(timestamp_ms);
8120
8121 // Expect encoded resolutions to match the expected simulcast layers.
8122 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8123 EXPECT_THAT(video_source_.sink_wants().resolutions,
8124 ::testing::ElementsAreArray({kLayer0Size, kLayer1Size}));
8125
8126 // Capture a frame with all but one layer turned off.
8127 timestamp_ms += kFrameIntervalMs;
8128 config.simulcast_layers[1].active = false;
8129 sink_.SetNumExpectedLayers(kNumSimulcastLayers - 2);
8130 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8131 video_source_.IncomingCapturedFrame(
8132 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8133 WaitForEncodedFrame(timestamp_ms);
8134
8135 // Expect encoded resolutions to match the expected simulcast layers.
8136 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8137 EXPECT_THAT(video_source_.sink_wants().resolutions,
8138 ::testing::ElementsAreArray({kLayer0Size}));
8139
8140 video_stream_encoder_->Stop();
8141}
8142
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008143TEST_F(VideoStreamEncoderTest, QpPresent_QpKept) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008144 ResetEncoder("VP8", 1, 1, 1, false);
8145
Niels Möller8b692902021-06-14 12:04:57 +02008146 // Force encoder reconfig.
8147 video_source_.IncomingCapturedFrame(
8148 CreateFrame(1, codec_width_, codec_height_));
8149 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8150
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008151 // Set QP on encoded frame and pass the frame to encode complete callback.
8152 // Since QP is present QP parsing won't be triggered and the original value
8153 // should be kept.
8154 EncodedImage encoded_image;
8155 encoded_image.qp_ = 123;
8156 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8157 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8158 CodecSpecificInfo codec_info;
8159 codec_info.codecType = kVideoCodecVP8;
8160 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8161 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8162 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 123);
8163 video_stream_encoder_->Stop();
8164}
8165
8166TEST_F(VideoStreamEncoderTest, QpAbsent_QpParsed) {
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008167 ResetEncoder("VP8", 1, 1, 1, false);
8168
Niels Möller8b692902021-06-14 12:04:57 +02008169 // Force encoder reconfig.
8170 video_source_.IncomingCapturedFrame(
8171 CreateFrame(1, codec_width_, codec_height_));
8172 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8173
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008174 // Pass an encoded frame without QP to encode complete callback. QP should be
8175 // parsed and set.
8176 EncodedImage encoded_image;
8177 encoded_image.qp_ = -1;
8178 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8179 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8180 CodecSpecificInfo codec_info;
8181 codec_info.codecType = kVideoCodecVP8;
8182 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8183 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8184 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, 25);
8185 video_stream_encoder_->Stop();
8186}
8187
8188TEST_F(VideoStreamEncoderTest, QpAbsentParsingDisabled_QpAbsent) {
Jonas Orelandc7f691a2022-03-09 15:12:07 +01008189 webrtc::test::ScopedKeyValueConfig field_trials(
8190 field_trials_, "WebRTC-QpParsingKillSwitch/Enabled/");
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008191
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008192 ResetEncoder("VP8", 1, 1, 1, false);
8193
Niels Möller8b692902021-06-14 12:04:57 +02008194 // Force encoder reconfig.
8195 video_source_.IncomingCapturedFrame(
8196 CreateFrame(1, codec_width_, codec_height_));
8197 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8198
Sergey Silkin0e42cf72021-03-15 10:12:57 +01008199 EncodedImage encoded_image;
8200 encoded_image.qp_ = -1;
8201 encoded_image.SetEncodedData(EncodedImageBuffer::Create(
8202 kCodedFrameVp8Qp25, sizeof(kCodedFrameVp8Qp25)));
8203 CodecSpecificInfo codec_info;
8204 codec_info.codecType = kVideoCodecVP8;
8205 fake_encoder_.InjectEncodedImage(encoded_image, &codec_info);
8206 EXPECT_TRUE(sink_.WaitForFrame(kDefaultTimeoutMs));
8207 EXPECT_EQ(sink_.GetLastEncodedImage().qp_, -1);
8208 video_stream_encoder_->Stop();
8209}
8210
Sergey Silkind19e3b92021-03-16 10:05:30 +00008211TEST_F(VideoStreamEncoderTest,
8212 QualityScalingNotAllowed_QualityScalingDisabled) {
8213 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8214
8215 // Disable scaling settings in encoder info.
8216 fake_encoder_.SetQualityScaling(false);
8217 // Disable quality scaling in encoder config.
8218 video_encoder_config.is_quality_scaling_allowed = false;
8219 ConfigureEncoder(std::move(video_encoder_config));
8220
8221 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008222 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008223
8224 test::FrameForwarder source;
8225 video_stream_encoder_->SetSource(
8226 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8227 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8228 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8229
8230 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8231 WaitForEncodedFrame(1);
8232 video_stream_encoder_->TriggerQualityLow();
8233 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8234
8235 video_stream_encoder_->Stop();
8236}
8237
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008238TEST_F(VideoStreamEncoderTest, QualityScalingNotAllowed_IsQpTrustedSetTrue) {
8239 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8240
8241 // Disable scaling settings in encoder info.
8242 fake_encoder_.SetQualityScaling(false);
8243 // Set QP trusted in encoder info.
8244 fake_encoder_.SetIsQpTrusted(true);
8245 // Enable quality scaling in encoder config.
8246 video_encoder_config.is_quality_scaling_allowed = false;
8247 ConfigureEncoder(std::move(video_encoder_config));
8248
8249 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008250 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008251
8252 test::FrameForwarder source;
8253 video_stream_encoder_->SetSource(
8254 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8255 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8256 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8257
8258 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8259 WaitForEncodedFrame(1);
8260 video_stream_encoder_->TriggerQualityLow();
8261 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8262
8263 video_stream_encoder_->Stop();
8264}
8265
Shuhai Pengf2707702021-09-29 17:19:44 +08008266TEST_F(VideoStreamEncoderTest,
8267 QualityScalingNotAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8268 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8269
8270 // Disable scaling settings in encoder info.
8271 fake_encoder_.SetQualityScaling(false);
8272 // Set QP trusted in encoder info.
8273 fake_encoder_.SetIsQpTrusted(true);
8274 // Enable quality scaling in encoder config.
8275 video_encoder_config.is_quality_scaling_allowed = false;
8276 ConfigureEncoder(std::move(video_encoder_config));
8277
8278 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008279 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008280
8281 test::FrameForwarder source;
8282 video_stream_encoder_->SetSource(
8283 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8284 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8285 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8286
8287 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8288 WaitForEncodedFrame(1);
8289 video_stream_encoder_->TriggerQualityLow();
8290 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8291
8292 video_stream_encoder_->Stop();
8293}
8294
8295TEST_F(VideoStreamEncoderTest,
8296 QualityScalingNotAllowedAndQPIsNotTrusted_BandwidthScalerDisable) {
8297 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8298
8299 // Disable scaling settings in encoder info.
8300 fake_encoder_.SetQualityScaling(false);
8301 // Set QP trusted in encoder info.
8302 fake_encoder_.SetIsQpTrusted(false);
8303 // Enable quality scaling in encoder config.
8304 video_encoder_config.is_quality_scaling_allowed = false;
8305 ConfigureEncoder(std::move(video_encoder_config));
8306
8307 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008308 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008309
8310 test::FrameForwarder source;
8311 video_stream_encoder_->SetSource(
8312 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8313 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8314 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8315
8316 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8317 WaitForEncodedFrame(1);
8318 video_stream_encoder_->TriggerQualityLow();
8319 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8320
8321 video_stream_encoder_->Stop();
8322}
8323
8324TEST_F(VideoStreamEncoderTest, EncoderProvideLimitsWhenQPIsNotTrusted) {
8325 // Set QP trusted in encoder info.
8326 fake_encoder_.SetIsQpTrusted(false);
8327
8328 const int MinEncBitrateKbps = 30;
8329 const int MaxEncBitrateKbps = 100;
8330 const int MinStartBitrateKbp = 50;
8331 const VideoEncoder::ResolutionBitrateLimits encoder_bitrate_limits(
8332 /*frame_size_pixels=*/codec_width_ * codec_height_,
8333 /*min_start_bitrate_bps=*/MinStartBitrateKbp,
8334 /*min_bitrate_bps=*/MinEncBitrateKbps * 1000,
8335 /*max_bitrate_bps=*/MaxEncBitrateKbps * 1000);
8336
8337 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008338 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008339
8340 fake_encoder_.SetResolutionBitrateLimits({encoder_bitrate_limits});
8341
8342 VideoEncoderConfig video_encoder_config;
8343 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8344 video_encoder_config.max_bitrate_bps = MaxEncBitrateKbps * 1000;
8345 video_encoder_config.simulcast_layers[0].min_bitrate_bps =
8346 MinEncBitrateKbps * 1000;
8347 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8348 kMaxPayloadLength);
8349
8350 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8351 WaitForEncodedFrame(1);
8352 EXPECT_EQ(
8353 MaxEncBitrateKbps,
8354 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8355 EXPECT_EQ(
8356 MinEncBitrateKbps,
8357 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8358
8359 video_stream_encoder_->Stop();
8360}
8361
8362TEST_F(VideoStreamEncoderTest, EncoderDoesnotProvideLimitsWhenQPIsNotTrusted) {
8363 // Set QP trusted in encoder info.
8364 fake_encoder_.SetIsQpTrusted(false);
8365
8366 absl::optional<VideoEncoder::ResolutionBitrateLimits> suitable_bitrate_limit =
8367 EncoderInfoSettings::
8368 GetSinglecastBitrateLimitForResolutionWhenQpIsUntrusted(
8369 codec_width_ * codec_height_,
8370 EncoderInfoSettings::
8371 GetDefaultSinglecastBitrateLimitsWhenQpIsUntrusted());
8372 EXPECT_TRUE(suitable_bitrate_limit.has_value());
8373
8374 const int MaxEncBitrate = suitable_bitrate_limit->max_bitrate_bps;
8375 const int MinEncBitrate = suitable_bitrate_limit->min_bitrate_bps;
8376 const int TargetEncBitrate = MaxEncBitrate;
8377 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8378 DataRate::BitsPerSec(TargetEncBitrate),
8379 DataRate::BitsPerSec(TargetEncBitrate),
8380 DataRate::BitsPerSec(TargetEncBitrate), 0, 0, 0);
8381
8382 VideoEncoderConfig video_encoder_config;
8383 test::FillEncoderConfiguration(kVideoCodecH264, 1, &video_encoder_config);
8384 video_encoder_config.max_bitrate_bps = MaxEncBitrate;
8385 video_encoder_config.simulcast_layers[0].min_bitrate_bps = MinEncBitrate;
8386 video_stream_encoder_->ConfigureEncoder(video_encoder_config.Copy(),
8387 kMaxPayloadLength);
8388
8389 video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr));
8390 WaitForEncodedFrame(1);
8391 EXPECT_EQ(
8392 MaxEncBitrate / 1000,
8393 static_cast<int>(bitrate_allocator_factory_.codec_config().maxBitrate));
8394 EXPECT_EQ(
8395 MinEncBitrate / 1000,
8396 static_cast<int>(bitrate_allocator_factory_.codec_config().minBitrate));
8397
8398 video_stream_encoder_->Stop();
8399}
8400
Erik Språnge4589cb2022-04-06 16:44:30 +02008401TEST_F(VideoStreamEncoderTest, NormalComplexityWithMoreThanTwoCores) {
8402 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8403 /*num_spatial_layers=*/1,
8404 /*screenshare=*/false, /*allocation_callback_type=*/
8405 VideoStreamEncoder::BitrateAllocationCallbackType::
8406 kVideoBitrateAllocationWhenScreenSharing,
8407 /*num_cores=*/3);
8408
8409 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8410 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8411 video_source_.IncomingCapturedFrame(
8412 CreateFrame(1, /*width=*/320, /*height=*/180));
8413 WaitForEncodedFrame(1);
8414 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8415 VideoCodecComplexity::kComplexityNormal);
8416 video_stream_encoder_->Stop();
8417}
8418
8419TEST_F(VideoStreamEncoderTest,
8420 NormalComplexityWhenLowTierOptimizationsAreDisabled) {
8421 webrtc::test::ScopedKeyValueConfig field_trials(
8422 field_trials_, "WebRTC-VP9-LowTierOptimizations/Disabled/");
8423
8424 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8425 /*num_spatial_layers=*/1,
8426 /*screenshare=*/false, /*allocation_callback_type=*/
8427 VideoStreamEncoder::BitrateAllocationCallbackType::
8428 kVideoBitrateAllocationWhenScreenSharing,
8429 /*num_cores=*/2);
8430
8431 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8432 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8433 video_source_.IncomingCapturedFrame(
8434 CreateFrame(1, /*width=*/320, /*height=*/180));
8435 WaitForEncodedFrame(1);
8436 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8437 VideoCodecComplexity::kComplexityNormal);
8438 video_stream_encoder_->Stop();
8439}
8440
8441TEST_F(VideoStreamEncoderTest, LowComplexityWithTwoCores) {
8442 ResetEncoder("VP9", /*num_stream=*/1, /*num_temporal_layers=*/1,
8443 /*num_spatial_layers=*/1,
8444 /*screenshare=*/false, /*allocation_callback_type=*/
8445 VideoStreamEncoder::BitrateAllocationCallbackType::
8446 kVideoBitrateAllocationWhenScreenSharing,
8447 /*num_cores=*/2);
8448
8449 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8450 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
8451 video_source_.IncomingCapturedFrame(
8452 CreateFrame(1, /*width=*/320, /*height=*/180));
8453 WaitForEncodedFrame(1);
8454 EXPECT_EQ(fake_encoder_.LastEncoderComplexity(),
8455 VideoCodecComplexity::kComplexityLow);
8456 video_stream_encoder_->Stop();
8457}
8458
Sergey Silkind19e3b92021-03-16 10:05:30 +00008459#if !defined(WEBRTC_IOS)
8460// TODO(bugs.webrtc.org/12401): Disabled because WebRTC-Video-QualityScaling is
8461// disabled by default on iOS.
8462TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_QualityScalingEnabled) {
8463 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8464
8465 // Disable scaling settings in encoder info.
8466 fake_encoder_.SetQualityScaling(false);
8467 // Enable quality scaling in encoder config.
8468 video_encoder_config.is_quality_scaling_allowed = true;
8469 ConfigureEncoder(std::move(video_encoder_config));
8470
8471 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008472 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Sergey Silkind19e3b92021-03-16 10:05:30 +00008473
8474 test::FrameForwarder source;
8475 video_stream_encoder_->SetSource(
8476 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8477 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8478 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8479
8480 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8481 WaitForEncodedFrame(1);
8482 video_stream_encoder_->TriggerQualityLow();
8483 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8484
8485 video_stream_encoder_->Stop();
8486}
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008487
8488TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetTrue) {
8489 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8490
8491 // Disable scaling settings in encoder info.
8492 fake_encoder_.SetQualityScaling(false);
8493 // Set QP trusted in encoder info.
8494 fake_encoder_.SetIsQpTrusted(true);
8495 // Enable quality scaling in encoder config.
8496 video_encoder_config.is_quality_scaling_allowed = true;
8497 ConfigureEncoder(std::move(video_encoder_config));
8498
8499 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008500 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Qiu Jianlinb54cfde2021-07-30 06:48:03 +08008501
8502 test::FrameForwarder source;
8503 video_stream_encoder_->SetSource(
8504 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8505 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8506 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8507
8508 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8509 WaitForEncodedFrame(1);
8510 video_stream_encoder_->TriggerQualityLow();
8511 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8512
8513 video_stream_encoder_->Stop();
8514}
Shuhai Pengf2707702021-09-29 17:19:44 +08008515
8516TEST_F(VideoStreamEncoderTest, QualityScalingAllowed_IsQpTrustedSetFalse) {
8517 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8518
8519 // Disable scaling settings in encoder info.
8520 fake_encoder_.SetQualityScaling(false);
8521 // Set QP not trusted in encoder info.
8522 fake_encoder_.SetIsQpTrusted(false);
8523 // Enable quality scaling in encoder config.
8524 video_encoder_config.is_quality_scaling_allowed = true;
8525 ConfigureEncoder(std::move(video_encoder_config));
8526
8527 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008528 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008529
8530 test::FrameForwarder source;
8531 video_stream_encoder_->SetSource(
8532 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8533 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8534 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8535
8536 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8537 WaitForEncodedFrame(1);
8538 video_stream_encoder_->TriggerQualityLow();
8539 // When quality_scaler doesn't work and is_quality_scaling_allowed is
8540 // true,the bandwidth_quality_scaler_ works,so bw_limited_resolution is true.
8541 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8542
8543 video_stream_encoder_->Stop();
8544}
8545
8546TEST_F(VideoStreamEncoderTest,
8547 QualityScalingAllowedAndQPIsTrusted_BandwidthScalerDisable) {
8548 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8549
8550 // Disable scaling settings in encoder info.
8551 fake_encoder_.SetQualityScaling(false);
8552 // Set QP trusted in encoder info.
8553 fake_encoder_.SetIsQpTrusted(true);
8554 // Enable quality scaling in encoder config.
8555 video_encoder_config.is_quality_scaling_allowed = true;
8556 ConfigureEncoder(std::move(video_encoder_config));
8557
8558 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008559 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008560
8561 test::FrameForwarder source;
8562 video_stream_encoder_->SetSource(
8563 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8564 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8565 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8566
8567 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8568 WaitForEncodedFrame(1);
8569 video_stream_encoder_->TriggerQualityLow();
8570 // bandwidth_quality_scaler isn't working, but quality_scaler is working.
8571 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8572
8573 video_stream_encoder_->Stop();
8574}
8575
8576TEST_F(VideoStreamEncoderTest,
8577 QualityScalingAllowedAndQPIsNotTrusted_BandwidthScalerEnabled) {
8578 VideoEncoderConfig video_encoder_config = video_encoder_config_.Copy();
8579
8580 // Disable scaling settings in encoder info.
8581 fake_encoder_.SetQualityScaling(false);
8582 // Set QP trusted in encoder info.
8583 fake_encoder_.SetIsQpTrusted(false);
8584 // Enable quality scaling in encoder config.
8585 video_encoder_config.is_quality_scaling_allowed = true;
8586 ConfigureEncoder(std::move(video_encoder_config));
8587
8588 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008589 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Shuhai Pengf2707702021-09-29 17:19:44 +08008590
8591 test::FrameForwarder source;
8592 video_stream_encoder_->SetSource(
8593 &source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
8594 EXPECT_THAT(source.sink_wants(), UnlimitedSinkWants());
8595 EXPECT_FALSE(stats_proxy_->GetStats().bw_limited_resolution);
8596
8597 source.IncomingCapturedFrame(CreateFrame(1, 1280, 720));
8598 WaitForEncodedFrame(1);
8599 video_stream_encoder_->TriggerQualityLow();
8600 EXPECT_TRUE(stats_proxy_->GetStats().bw_limited_resolution);
8601
8602 video_stream_encoder_->Stop();
8603}
8604
Erik Språnge4589cb2022-04-06 16:44:30 +02008605#endif // !defined(WEBRTC_IOS)
Sergey Silkind19e3b92021-03-16 10:05:30 +00008606
Henrik Boström56db9ff2021-03-24 09:06:45 +01008607// Test parameters: (VideoCodecType codec, bool allow_i420_conversion)
8608class VideoStreamEncoderWithRealEncoderTest
8609 : public VideoStreamEncoderTest,
8610 public ::testing::WithParamInterface<std::pair<VideoCodecType, bool>> {
8611 public:
8612 VideoStreamEncoderWithRealEncoderTest()
8613 : VideoStreamEncoderTest(),
8614 codec_type_(std::get<0>(GetParam())),
8615 allow_i420_conversion_(std::get<1>(GetParam())) {}
8616
8617 void SetUp() override {
8618 VideoStreamEncoderTest::SetUp();
8619 std::unique_ptr<VideoEncoder> encoder;
8620 switch (codec_type_) {
8621 case kVideoCodecVP8:
8622 encoder = VP8Encoder::Create();
8623 break;
8624 case kVideoCodecVP9:
8625 encoder = VP9Encoder::Create();
8626 break;
8627 case kVideoCodecAV1:
philipel95701502022-01-18 18:47:52 +01008628 encoder = CreateLibaomAv1EncoderIfSupported();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008629 break;
8630 case kVideoCodecH264:
8631 encoder =
8632 H264Encoder::Create(cricket::VideoCodec(cricket::kH264CodecName));
8633 break;
8634 case kVideoCodecMultiplex:
8635 mock_encoder_factory_for_multiplex_ =
8636 std::make_unique<MockVideoEncoderFactory>();
8637 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, Die);
8638 EXPECT_CALL(*mock_encoder_factory_for_multiplex_, CreateVideoEncoder)
8639 .WillRepeatedly([] { return VP8Encoder::Create(); });
8640 encoder = std::make_unique<MultiplexEncoderAdapter>(
8641 mock_encoder_factory_for_multiplex_.get(), SdpVideoFormat("VP8"),
8642 false);
8643 break;
8644 default:
Artem Titovd3251962021-11-15 16:57:07 +01008645 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008646 }
8647 ConfigureEncoderAndBitrate(codec_type_, std::move(encoder));
8648 }
8649
8650 void TearDown() override {
8651 video_stream_encoder_->Stop();
Artem Titovab30d722021-07-27 16:22:11 +02008652 // Ensure `video_stream_encoder_` is destroyed before
8653 // `encoder_proxy_factory_`.
Henrik Boström56db9ff2021-03-24 09:06:45 +01008654 video_stream_encoder_.reset();
8655 VideoStreamEncoderTest::TearDown();
8656 }
8657
8658 protected:
8659 void ConfigureEncoderAndBitrate(VideoCodecType codec_type,
8660 std::unique_ptr<VideoEncoder> encoder) {
8661 // Configure VSE to use the encoder.
8662 encoder_ = std::move(encoder);
8663 encoder_proxy_factory_ = std::make_unique<test::VideoEncoderProxyFactory>(
8664 encoder_.get(), &encoder_selector_);
8665 video_send_config_.encoder_settings.encoder_factory =
8666 encoder_proxy_factory_.get();
8667 VideoEncoderConfig video_encoder_config;
8668 test::FillEncoderConfiguration(codec_type, 1, &video_encoder_config);
8669 video_encoder_config_ = video_encoder_config.Copy();
8670 ConfigureEncoder(video_encoder_config_.Copy());
8671
8672 // Set bitrate to ensure frame is not dropped.
8673 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008674 kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008675 }
8676
8677 const VideoCodecType codec_type_;
8678 const bool allow_i420_conversion_;
8679 NiceMock<MockEncoderSelector> encoder_selector_;
8680 std::unique_ptr<test::VideoEncoderProxyFactory> encoder_proxy_factory_;
8681 std::unique_ptr<VideoEncoder> encoder_;
8682 std::unique_ptr<MockVideoEncoderFactory> mock_encoder_factory_for_multiplex_;
8683};
8684
8685TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeI420) {
8686 auto native_i420_frame = test::CreateMappableNativeFrame(
8687 1, VideoFrameBuffer::Type::kI420, codec_width_, codec_height_);
8688 video_source_.IncomingCapturedFrame(native_i420_frame);
8689 WaitForEncodedFrame(codec_width_, codec_height_);
8690
8691 auto mappable_native_buffer =
8692 test::GetMappableNativeBufferFromVideoFrame(native_i420_frame);
8693 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8694 mappable_native_buffer->GetMappedFramedBuffers();
8695 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8696 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8697 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8698 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kI420);
8699}
8700
8701TEST_P(VideoStreamEncoderWithRealEncoderTest, EncoderMapsNativeNV12) {
8702 auto native_nv12_frame = test::CreateMappableNativeFrame(
8703 1, VideoFrameBuffer::Type::kNV12, codec_width_, codec_height_);
8704 video_source_.IncomingCapturedFrame(native_nv12_frame);
8705 WaitForEncodedFrame(codec_width_, codec_height_);
8706
8707 auto mappable_native_buffer =
8708 test::GetMappableNativeBufferFromVideoFrame(native_nv12_frame);
8709 std::vector<rtc::scoped_refptr<VideoFrameBuffer>> mapped_frame_buffers =
8710 mappable_native_buffer->GetMappedFramedBuffers();
8711 ASSERT_EQ(mapped_frame_buffers.size(), 1u);
8712 EXPECT_EQ(mapped_frame_buffers[0]->width(), codec_width_);
8713 EXPECT_EQ(mapped_frame_buffers[0]->height(), codec_height_);
8714 EXPECT_EQ(mapped_frame_buffers[0]->type(), VideoFrameBuffer::Type::kNV12);
8715
8716 if (!allow_i420_conversion_) {
8717 EXPECT_FALSE(mappable_native_buffer->DidConvertToI420());
8718 }
8719}
8720
Erik Språng7444b192021-06-02 14:02:13 +02008721TEST_P(VideoStreamEncoderWithRealEncoderTest, HandlesLayerToggling) {
8722 if (codec_type_ == kVideoCodecMultiplex) {
8723 // Multiplex codec here uses wrapped mock codecs, ignore for this test.
8724 return;
8725 }
8726
8727 const size_t kNumSpatialLayers = 3u;
8728 const float kDownscaleFactors[] = {4.0, 2.0, 1.0};
8729 const int kFrameWidth = 1280;
8730 const int kFrameHeight = 720;
8731 const rtc::VideoSinkWants::FrameSize kLayer0Size(
8732 kFrameWidth / kDownscaleFactors[0], kFrameHeight / kDownscaleFactors[0]);
8733 const rtc::VideoSinkWants::FrameSize kLayer1Size(
8734 kFrameWidth / kDownscaleFactors[1], kFrameHeight / kDownscaleFactors[1]);
8735 const rtc::VideoSinkWants::FrameSize kLayer2Size(
8736 kFrameWidth / kDownscaleFactors[2], kFrameHeight / kDownscaleFactors[2]);
8737
8738 VideoEncoderConfig config;
8739 if (codec_type_ == VideoCodecType::kVideoCodecVP9) {
8740 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008741 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008742 VideoCodecVP9 vp9_settings = VideoEncoder::GetDefaultVp9Settings();
8743 vp9_settings.numberOfSpatialLayers = kNumSpatialLayers;
8744 vp9_settings.numberOfTemporalLayers = 3;
8745 vp9_settings.automaticResizeOn = false;
8746 config.encoder_specific_settings =
8747 rtc::make_ref_counted<VideoEncoderConfig::Vp9EncoderSpecificSettings>(
8748 vp9_settings);
8749 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8750 /*fps=*/30.0,
8751 /*first_active_layer=*/0,
8752 /*num_spatial_layers=*/3,
8753 /*num_temporal_layers=*/3,
8754 /*is_screenshare=*/false);
8755 } else if (codec_type_ == VideoCodecType::kVideoCodecAV1) {
8756 test::FillEncoderConfiguration(codec_type_, 1, &config);
Asa Persson606d3cb2021-10-04 10:07:11 +02008757 config.max_bitrate_bps = kSimulcastTargetBitrate.bps();
Erik Språng7444b192021-06-02 14:02:13 +02008758 config.spatial_layers = GetSvcConfig(kFrameWidth, kFrameHeight,
8759 /*fps=*/30.0,
8760 /*first_active_layer=*/0,
8761 /*num_spatial_layers=*/3,
8762 /*num_temporal_layers=*/3,
8763 /*is_screenshare=*/false);
8764 config.simulcast_layers[0].scalability_mode = "L3T3_KEY";
8765 } else {
8766 // Simulcast for VP8/H264.
8767 test::FillEncoderConfiguration(codec_type_, kNumSpatialLayers, &config);
8768 for (size_t i = 0; i < kNumSpatialLayers; ++i) {
8769 config.simulcast_layers[i].scale_resolution_down_by =
8770 kDownscaleFactors[i];
8771 config.simulcast_layers[i].active = true;
8772 }
8773 if (codec_type_ == VideoCodecType::kVideoCodecH264) {
8774 // Turn off frame dropping to prevent flakiness.
8775 VideoCodecH264 h264_settings = VideoEncoder::GetDefaultH264Settings();
8776 h264_settings.frameDroppingOn = false;
8777 config.encoder_specific_settings = rtc::make_ref_counted<
8778 VideoEncoderConfig::H264EncoderSpecificSettings>(h264_settings);
8779 }
8780 }
8781
8782 auto set_layer_active = [&](int layer_idx, bool active) {
8783 if (codec_type_ == VideoCodecType::kVideoCodecVP9 ||
8784 codec_type_ == VideoCodecType::kVideoCodecAV1) {
8785 config.spatial_layers[layer_idx].active = active;
8786 } else {
8787 config.simulcast_layers[layer_idx].active = active;
8788 }
8789 };
8790
8791 config.video_stream_factory =
8792 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8793 CodecTypeToPayloadString(codec_type_), /*max qp*/ 56,
8794 /*screencast*/ false,
8795 /*screenshare enabled*/ false);
8796 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
Asa Persson606d3cb2021-10-04 10:07:11 +02008797 kSimulcastTargetBitrate, kSimulcastTargetBitrate, kSimulcastTargetBitrate,
8798 0, 0, 0);
Erik Språng7444b192021-06-02 14:02:13 +02008799
8800 // Capture a frame with all layers active.
8801 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8802 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8803 int64_t timestamp_ms = kFrameIntervalMs;
8804 video_source_.IncomingCapturedFrame(
8805 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8806
8807 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8808 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8809
8810 // Capture a frame with one of the layers inactive.
8811 set_layer_active(2, false);
8812 sink_.SetNumExpectedLayers(kNumSpatialLayers - 1);
8813 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8814 timestamp_ms += kFrameIntervalMs;
8815 video_source_.IncomingCapturedFrame(
8816 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8817 WaitForEncodedFrame(kLayer1Size.width, kLayer1Size.height);
8818
8819 // New target bitrates signaled based on lower resolution.
8820 DataRate kTwoLayerBitrate = DataRate::KilobitsPerSec(833);
8821 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8822 kTwoLayerBitrate, kTwoLayerBitrate, kTwoLayerBitrate, 0, 0, 0);
8823 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8824
8825 // Re-enable the top layer.
8826 set_layer_active(2, true);
8827 sink_.SetNumExpectedLayers(kNumSpatialLayers);
8828 video_stream_encoder_->ConfigureEncoder(config.Copy(), kMaxPayloadLength);
8829 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8830
8831 // Bitrate target adjusted back up to enable HD layer...
8832 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8833 DataRate::KilobitsPerSec(1800), DataRate::KilobitsPerSec(1800),
8834 DataRate::KilobitsPerSec(1800), 0, 0, 0);
8835 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8836
8837 // ...then add a new frame.
8838 timestamp_ms += kFrameIntervalMs;
8839 video_source_.IncomingCapturedFrame(
8840 CreateFrame(timestamp_ms, kFrameWidth, kFrameHeight));
8841 WaitForEncodedFrame(kLayer2Size.width, kLayer2Size.height);
8842 video_stream_encoder_->WaitUntilTaskQueueIsIdle();
8843
8844 video_stream_encoder_->Stop();
8845}
8846
Henrik Boström56db9ff2021-03-24 09:06:45 +01008847std::string TestParametersVideoCodecAndAllowI420ConversionToString(
8848 testing::TestParamInfo<std::pair<VideoCodecType, bool>> info) {
8849 VideoCodecType codec_type = std::get<0>(info.param);
8850 bool allow_i420_conversion = std::get<1>(info.param);
8851 std::string str;
8852 switch (codec_type) {
8853 case kVideoCodecGeneric:
8854 str = "Generic";
8855 break;
8856 case kVideoCodecVP8:
8857 str = "VP8";
8858 break;
8859 case kVideoCodecVP9:
8860 str = "VP9";
8861 break;
8862 case kVideoCodecAV1:
8863 str = "AV1";
8864 break;
8865 case kVideoCodecH264:
8866 str = "H264";
8867 break;
8868 case kVideoCodecMultiplex:
8869 str = "Multiplex";
8870 break;
8871 default:
Artem Titovd3251962021-11-15 16:57:07 +01008872 RTC_DCHECK_NOTREACHED();
Henrik Boström56db9ff2021-03-24 09:06:45 +01008873 }
8874 str += allow_i420_conversion ? "_AllowToI420" : "_DisallowToI420";
8875 return str;
8876}
8877
8878constexpr std::pair<VideoCodecType, bool> kVP8DisallowConversion =
8879 std::make_pair(kVideoCodecVP8, /*allow_i420_conversion=*/false);
8880constexpr std::pair<VideoCodecType, bool> kVP9DisallowConversion =
8881 std::make_pair(kVideoCodecVP9, /*allow_i420_conversion=*/false);
8882constexpr std::pair<VideoCodecType, bool> kAV1AllowConversion =
Ilya Nikolaevskiy1bcdafc2022-03-09 16:01:07 +01008883 std::make_pair(kVideoCodecAV1, /*allow_i420_conversion=*/false);
Henrik Boström56db9ff2021-03-24 09:06:45 +01008884constexpr std::pair<VideoCodecType, bool> kMultiplexDisallowConversion =
8885 std::make_pair(kVideoCodecMultiplex, /*allow_i420_conversion=*/false);
8886#if defined(WEBRTC_USE_H264)
8887constexpr std::pair<VideoCodecType, bool> kH264AllowConversion =
8888 std::make_pair(kVideoCodecH264, /*allow_i420_conversion=*/true);
8889
8890// The windows compiler does not tolerate #if statements inside the
8891// INSTANTIATE_TEST_SUITE_P() macro, so we have to have two definitions (with
8892// and without H264).
8893INSTANTIATE_TEST_SUITE_P(
8894 All,
8895 VideoStreamEncoderWithRealEncoderTest,
8896 ::testing::Values(kVP8DisallowConversion,
8897 kVP9DisallowConversion,
8898 kAV1AllowConversion,
8899 kMultiplexDisallowConversion,
8900 kH264AllowConversion),
8901 TestParametersVideoCodecAndAllowI420ConversionToString);
8902#else
8903INSTANTIATE_TEST_SUITE_P(
8904 All,
8905 VideoStreamEncoderWithRealEncoderTest,
8906 ::testing::Values(kVP8DisallowConversion,
8907 kVP9DisallowConversion,
8908 kAV1AllowConversion,
8909 kMultiplexDisallowConversion),
8910 TestParametersVideoCodecAndAllowI420ConversionToString);
8911#endif
8912
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008913class ReconfigureEncoderTest : public VideoStreamEncoderTest {
8914 protected:
8915 void RunTest(const std::vector<VideoStream>& configs,
8916 const int expected_num_init_encode) {
8917 ConfigureEncoder(configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008918 OnBitrateUpdated(kTargetBitrate);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008919 InsertFrameAndWaitForEncoded();
8920 EXPECT_EQ(1, sink_.number_of_reconfigurations());
8921 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[0]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008922 EXPECT_EQ(1, fake_encoder_.GetNumInitializations());
8923 ExpectEqual(fake_encoder_.config(), configs[0]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008924
8925 // Reconfigure encoder, the encoder should only be reconfigured if needed.
8926 ConfigureEncoder(configs[1]);
8927 InsertFrameAndWaitForEncoded();
8928 EXPECT_EQ(2, sink_.number_of_reconfigurations());
8929 ExpectEqual(bitrate_allocator_factory_.codec_config(), configs[1]);
Asa Persson606d3cb2021-10-04 10:07:11 +02008930 EXPECT_EQ(expected_num_init_encode, fake_encoder_.GetNumInitializations());
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008931 if (expected_num_init_encode > 1)
Asa Persson606d3cb2021-10-04 10:07:11 +02008932 ExpectEqual(fake_encoder_.config(), configs[1]);
Åsa Persson4d4f62f2021-09-12 16:14:48 +02008933
8934 video_stream_encoder_->Stop();
8935 }
8936
8937 void ConfigureEncoder(const VideoStream& stream) {
8938 VideoEncoderConfig config;
8939 test::FillEncoderConfiguration(kVideoCodecVP8, /*num_streams=*/1, &config);
8940 config.max_bitrate_bps = stream.max_bitrate_bps;
8941 config.simulcast_layers[0] = stream;
8942 config.video_stream_factory =
8943 rtc::make_ref_counted<cricket::EncoderStreamFactory>(
8944 /*codec_name=*/"VP8", /*max_qp=*/0, /*is_screenshare=*/false,
8945 /*conference_mode=*/false);
8946 video_stream_encoder_->ConfigureEncoder(std::move(config),
8947 kMaxPayloadLength);
8948 }
8949
8950 void OnBitrateUpdated(DataRate bitrate) {
8951 video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
8952 bitrate, bitrate, bitrate, 0, 0, 0);
8953 }
8954
8955 void InsertFrameAndWaitForEncoded() {
8956 timestamp_ms_ += kFrameIntervalMs;
8957 video_source_.IncomingCapturedFrame(
8958 CreateFrame(timestamp_ms_, kWidth, kHeight));
8959 sink_.WaitForEncodedFrame(timestamp_ms_);
8960 }
8961
8962 void ExpectEqual(const VideoCodec& actual,
8963 const VideoStream& expected) const {
8964 EXPECT_EQ(actual.numberOfSimulcastStreams, 1);
8965 EXPECT_EQ(actual.simulcastStream[0].maxFramerate, expected.max_framerate);
8966 EXPECT_EQ(actual.simulcastStream[0].minBitrate * 1000,
8967 static_cast<unsigned int>(expected.min_bitrate_bps));
8968 EXPECT_EQ(actual.simulcastStream[0].maxBitrate * 1000,
8969 static_cast<unsigned int>(expected.max_bitrate_bps));
8970 EXPECT_EQ(actual.simulcastStream[0].width,
8971 kWidth / expected.scale_resolution_down_by);
8972 EXPECT_EQ(actual.simulcastStream[0].height,
8973 kHeight / expected.scale_resolution_down_by);
8974 EXPECT_EQ(actual.simulcastStream[0].numberOfTemporalLayers,
8975 expected.num_temporal_layers);
8976 EXPECT_EQ(actual.ScalabilityMode(), expected.scalability_mode);
8977 }
8978
8979 VideoStream DefaultConfig() const {
8980 VideoStream stream;
8981 stream.max_framerate = 25;
8982 stream.min_bitrate_bps = 35000;
8983 stream.max_bitrate_bps = 900000;
8984 stream.scale_resolution_down_by = 1.0;
8985 stream.num_temporal_layers = 1;
8986 stream.bitrate_priority = 1.0;
8987 stream.scalability_mode = "";
8988 return stream;
8989 }
8990
8991 const int kWidth = 640;
8992 const int kHeight = 360;
8993 int64_t timestamp_ms_ = 0;
8994};
8995
8996TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxFramerateChanges) {
8997 VideoStream config1 = DefaultConfig();
8998 VideoStream config2 = config1;
8999 config2.max_framerate++;
9000
9001 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9002}
9003
9004TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMinBitrateChanges) {
9005 VideoStream config1 = DefaultConfig();
9006 VideoStream config2 = config1;
9007 config2.min_bitrate_bps += 10000;
9008
9009 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9010}
9011
9012TEST_F(ReconfigureEncoderTest, NotReconfiguredIfMaxBitrateChanges) {
9013 VideoStream config1 = DefaultConfig();
9014 VideoStream config2 = config1;
9015 config2.max_bitrate_bps += 100000;
9016
9017 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9018}
9019
9020TEST_F(ReconfigureEncoderTest, NotReconfiguredIfBitratePriorityChanges) {
9021 VideoStream config1 = DefaultConfig();
9022 VideoStream config2 = config1;
9023 config2.bitrate_priority = config1.bitrate_priority.value() * 2.0;
9024
9025 RunTest({config1, config2}, /*expected_num_init_encode=*/1);
9026}
9027
9028TEST_F(ReconfigureEncoderTest, ReconfiguredIfResolutionChanges) {
9029 VideoStream config1 = DefaultConfig();
9030 VideoStream config2 = config1;
9031 config2.scale_resolution_down_by *= 2;
9032
9033 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9034}
9035
9036TEST_F(ReconfigureEncoderTest, ReconfiguredIfNumTemporalLayerChanges) {
9037 VideoStream config1 = DefaultConfig();
9038 VideoStream config2 = config1;
9039 config2.num_temporal_layers = config1.num_temporal_layers.value() + 1;
9040
9041 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9042}
9043
9044TEST_F(ReconfigureEncoderTest, ReconfiguredIfScalabilityModeChanges) {
9045 VideoStream config1 = DefaultConfig();
9046 VideoStream config2 = config1;
9047 config2.scalability_mode = "L1T2";
9048
9049 RunTest({config1, config2}, /*expected_num_init_encode=*/2);
9050}
9051
Tommi62b01db2022-01-25 23:41:22 +01009052// Simple test that just creates and then immediately destroys an encoder.
9053// The purpose of the test is to make sure that nothing bad happens if the
9054// initialization step on the encoder queue, doesn't run.
9055TEST(VideoStreamEncoderSimpleTest, CreateDestroy) {
9056 class SuperLazyTaskQueue : public webrtc::TaskQueueBase {
9057 public:
9058 SuperLazyTaskQueue() = default;
9059 ~SuperLazyTaskQueue() override = default;
9060
9061 private:
9062 void Delete() override { delete this; }
9063 void PostTask(std::unique_ptr<QueuedTask> task) override {
9064 // meh.
9065 }
9066 void PostDelayedTask(std::unique_ptr<QueuedTask> task,
9067 uint32_t milliseconds) override {
9068 ASSERT_TRUE(false);
9069 }
9070 };
9071
9072 // Lots of boiler plate.
Jonas Oreland8ca06132022-03-14 12:52:48 +01009073 test::ScopedKeyValueConfig field_trials;
Tommi62b01db2022-01-25 23:41:22 +01009074 GlobalSimulatedTimeController time_controller(Timestamp::Millis(0));
9075 auto stats_proxy = std::make_unique<MockableSendStatisticsProxy>(
9076 time_controller.GetClock(), VideoSendStream::Config(nullptr),
Jonas Oreland8ca06132022-03-14 12:52:48 +01009077 webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo, field_trials);
Tommi62b01db2022-01-25 23:41:22 +01009078 SimpleVideoStreamEncoderFactory::MockFakeEncoder mock_fake_encoder(
9079 time_controller.GetClock());
9080 test::VideoEncoderProxyFactory encoder_factory(&mock_fake_encoder);
9081 std::unique_ptr<VideoBitrateAllocatorFactory> bitrate_allocator_factory =
9082 CreateBuiltinVideoBitrateAllocatorFactory();
9083 VideoStreamEncoderSettings encoder_settings{
9084 VideoEncoder::Capabilities(/*loss_notification=*/false)};
9085 encoder_settings.encoder_factory = &encoder_factory;
9086 encoder_settings.bitrate_allocator_factory = bitrate_allocator_factory.get();
9087
9088 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9089 EXPECT_CALL((*adapter.get()), Initialize).WillOnce(Return());
9090
9091 std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter>
9092 encoder_queue(new SuperLazyTaskQueue());
9093
9094 // Construct a VideoStreamEncoder instance and let it go out of scope without
9095 // doing anything else (including calling Stop()). This should be fine since
9096 // the posted init task will simply be deleted.
9097 auto encoder = std::make_unique<VideoStreamEncoder>(
9098 time_controller.GetClock(), 1, stats_proxy.get(), encoder_settings,
9099 std::make_unique<CpuOveruseDetectorProxy>(stats_proxy.get()),
9100 std::move(adapter), std::move(encoder_queue),
9101 VideoStreamEncoder::BitrateAllocationCallbackType::
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009102 kVideoBitrateAllocation,
9103 field_trials);
Tommi62b01db2022-01-25 23:41:22 +01009104}
9105
Markus Handellb4e96d42021-11-05 12:00:55 +01009106TEST(VideoStreamEncoderFrameCadenceTest, ActivatesFrameCadenceOnContentType) {
9107 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9108 auto* adapter_ptr = adapter.get();
9109 SimpleVideoStreamEncoderFactory factory;
Markus Handell8d87c462021-12-16 11:37:16 +01009110 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9111 nullptr;
9112 EXPECT_CALL(*adapter_ptr, Initialize)
9113 .WillOnce(Invoke([&video_stream_encoder_callback](
9114 FrameCadenceAdapterInterface::Callback* callback) {
9115 video_stream_encoder_callback = callback;
9116 }));
9117 TaskQueueBase* encoder_queue = nullptr;
9118 auto video_stream_encoder =
9119 factory.Create(std::move(adapter), &encoder_queue);
Markus Handellb4e96d42021-11-05 12:00:55 +01009120
Markus Handelle59fee82021-12-23 09:29:23 +01009121 // First a call before we know the frame size and hence cannot compute the
9122 // number of simulcast layers.
9123 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
9124 &FrameCadenceAdapterInterface::
9125 ZeroHertzModeParams::num_simulcast_layers,
9126 Eq(0)))));
Markus Handellb4e96d42021-11-05 12:00:55 +01009127 VideoEncoderConfig config;
Markus Handell8d87c462021-12-16 11:37:16 +01009128 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
Markus Handellb4e96d42021-11-05 12:00:55 +01009129 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9130 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
Markus Handelle59fee82021-12-23 09:29:23 +01009131 factory.DepleteTaskQueues();
9132
9133 // Then a call as we've computed the number of simulcast layers after a passed
9134 // frame.
9135 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Optional(Field(
9136 &FrameCadenceAdapterInterface::
9137 ZeroHertzModeParams::num_simulcast_layers,
9138 Gt(0)))));
Markus Handell8d87c462021-12-16 11:37:16 +01009139 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handell9a478b52021-11-18 16:07:01 +01009140 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01009141 Mock::VerifyAndClearExpectations(adapter_ptr);
9142
Markus Handelle59fee82021-12-23 09:29:23 +01009143 // Expect a disabled zero-hertz mode after passing realtime video.
Markus Handell8d87c462021-12-16 11:37:16 +01009144 EXPECT_CALL(*adapter_ptr, SetZeroHertzModeEnabled(Eq(absl::nullopt)));
Markus Handellb4e96d42021-11-05 12:00:55 +01009145 VideoEncoderConfig config2;
Markus Handell8d87c462021-12-16 11:37:16 +01009146 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config2);
Markus Handellb4e96d42021-11-05 12:00:55 +01009147 config2.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
9148 video_stream_encoder->ConfigureEncoder(std::move(config2), 0);
Markus Handell8d87c462021-12-16 11:37:16 +01009149 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
Markus Handell9a478b52021-11-18 16:07:01 +01009150 factory.DepleteTaskQueues();
Markus Handellb4e96d42021-11-05 12:00:55 +01009151}
9152
9153TEST(VideoStreamEncoderFrameCadenceTest,
9154 ForwardsFramesIntoFrameCadenceAdapter) {
9155 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9156 auto* adapter_ptr = adapter.get();
9157 test::FrameForwarder video_source;
9158 SimpleVideoStreamEncoderFactory factory;
9159 auto video_stream_encoder = factory.Create(std::move(adapter));
9160 video_stream_encoder->SetSource(
9161 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9162
9163 EXPECT_CALL(*adapter_ptr, OnFrame);
9164 auto buffer = rtc::make_ref_counted<NV12Buffer>(/*width=*/16, /*height=*/16);
9165 video_source.IncomingCapturedFrame(
Markus Handell8d87c462021-12-16 11:37:16 +01009166 VideoFrame::Builder().set_video_frame_buffer(std::move(buffer)).build());
Markus Handellb4e96d42021-11-05 12:00:55 +01009167}
9168
Markus Handellee225432021-11-29 12:35:12 +01009169TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) {
9170 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9171 auto* adapter_ptr = adapter.get();
9172 test::FrameForwarder video_source;
9173 SimpleVideoStreamEncoderFactory factory;
9174 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9175 nullptr;
9176 EXPECT_CALL(*adapter_ptr, Initialize)
9177 .WillOnce(Invoke([&video_stream_encoder_callback](
9178 FrameCadenceAdapterInterface::Callback* callback) {
9179 video_stream_encoder_callback = callback;
9180 }));
9181 TaskQueueBase* encoder_queue = nullptr;
9182 auto video_stream_encoder =
9183 factory.Create(std::move(adapter), &encoder_queue);
9184
9185 // This is just to make the VSE operational. We'll feed a frame directly by
9186 // the callback interface.
9187 video_stream_encoder->SetSource(
9188 &video_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9189
9190 VideoEncoderConfig video_encoder_config;
9191 test::FillEncoderConfiguration(kVideoCodecGeneric, 1, &video_encoder_config);
9192 video_stream_encoder->ConfigureEncoder(std::move(video_encoder_config),
9193 /*max_data_payload_length=*/1000);
9194
9195 EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps);
9196 EXPECT_CALL(*adapter_ptr, UpdateFrameRate);
Markus Handell8d87c462021-12-16 11:37:16 +01009197 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
Markus Handellee225432021-11-29 12:35:12 +01009198 factory.DepleteTaskQueues();
9199}
9200
Markus Handell8d87c462021-12-16 11:37:16 +01009201TEST(VideoStreamEncoderFrameCadenceTest,
9202 DeactivatesActivatesLayersOnBitrateChanges) {
9203 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9204 auto* adapter_ptr = adapter.get();
9205 SimpleVideoStreamEncoderFactory factory;
9206 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9207 nullptr;
9208 EXPECT_CALL(*adapter_ptr, Initialize)
9209 .WillOnce(Invoke([&video_stream_encoder_callback](
9210 FrameCadenceAdapterInterface::Callback* callback) {
9211 video_stream_encoder_callback = callback;
9212 }));
9213 TaskQueueBase* encoder_queue = nullptr;
9214 auto video_stream_encoder =
9215 factory.Create(std::move(adapter), &encoder_queue);
9216
9217 // Configure 2 simulcast layers. FillEncoderConfiguration sets min bitrates to
9218 // {150000, 450000}.
9219 VideoEncoderConfig video_encoder_config;
9220 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9221 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9222 kMaxPayloadLength);
9223 // Ensure an encoder is created.
9224 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9225
9226 // Both layers enabled at 1 MBit/s.
9227 video_stream_encoder->OnBitrateUpdated(
9228 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9229 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9230 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9231 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9232 factory.DepleteTaskQueues();
9233 Mock::VerifyAndClearExpectations(adapter_ptr);
9234
9235 // Layer 1 disabled at 200 KBit/s.
9236 video_stream_encoder->OnBitrateUpdated(
9237 DataRate::KilobitsPerSec(200), DataRate::KilobitsPerSec(200),
9238 DataRate::KilobitsPerSec(200), 0, 0, 0);
9239 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9240 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9241 factory.DepleteTaskQueues();
9242 Mock::VerifyAndClearExpectations(adapter_ptr);
9243
9244 // All layers off at suspended video.
9245 video_stream_encoder->OnBitrateUpdated(DataRate::Zero(), DataRate::Zero(),
9246 DataRate::Zero(), 0, 0, 0);
9247 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/false));
9248 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/false));
9249 factory.DepleteTaskQueues();
9250 Mock::VerifyAndClearExpectations(adapter_ptr);
9251
9252 // Both layers enabled again back at 1 MBit/s.
9253 video_stream_encoder->OnBitrateUpdated(
9254 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9255 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9256 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(0, /*enabled=*/true));
9257 EXPECT_CALL(*adapter_ptr, UpdateLayerStatus(1, /*enabled=*/true));
9258 factory.DepleteTaskQueues();
9259}
9260
9261TEST(VideoStreamEncoderFrameCadenceTest, UpdatesQualityConvergence) {
9262 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9263 auto* adapter_ptr = adapter.get();
9264 SimpleVideoStreamEncoderFactory factory;
9265 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9266 nullptr;
9267 EXPECT_CALL(*adapter_ptr, Initialize)
9268 .WillOnce(Invoke([&video_stream_encoder_callback](
9269 FrameCadenceAdapterInterface::Callback* callback) {
9270 video_stream_encoder_callback = callback;
9271 }));
9272 TaskQueueBase* encoder_queue = nullptr;
9273 auto video_stream_encoder =
9274 factory.Create(std::move(adapter), &encoder_queue);
9275
9276 // Configure 2 simulcast layers and setup 1 MBit/s to unpause the encoder.
9277 VideoEncoderConfig video_encoder_config;
9278 test::FillEncoderConfiguration(kVideoCodecVP8, 2, &video_encoder_config);
9279 video_stream_encoder->ConfigureEncoder(video_encoder_config.Copy(),
9280 kMaxPayloadLength);
9281 video_stream_encoder->OnBitrateUpdated(
9282 DataRate::KilobitsPerSec(1000), DataRate::KilobitsPerSec(1000),
9283 DataRate::KilobitsPerSec(1000), 0, 0, 0);
9284
9285 // Pass a frame which has unconverged results.
9286 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
9287 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9288 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9289 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
9290 EXPECT_FALSE(encoded_image.IsAtTargetQuality());
9291 CodecSpecificInfo codec_specific;
9292 codec_specific.codecType = kVideoCodecGeneric;
9293 return codec_specific;
9294 }));
9295 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, false));
9296 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9297 factory.DepleteTaskQueues();
9298 Mock::VerifyAndClearExpectations(adapter_ptr);
9299 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9300
9301 // Pass a frame which converges in layer 0 and not in layer 1.
9302 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9303 EXPECT_CALL(factory.GetMockFakeEncoder(), EncodeHook)
9304 .WillRepeatedly(Invoke([](EncodedImage& encoded_image,
9305 rtc::scoped_refptr<EncodedImageBuffer> buffer) {
9306 encoded_image.SetAtTargetQuality(encoded_image.SpatialIndex() == 0);
9307 CodecSpecificInfo codec_specific;
9308 codec_specific.codecType = kVideoCodecGeneric;
9309 return codec_specific;
9310 }));
9311 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(0, true));
9312 EXPECT_CALL(*adapter_ptr, UpdateLayerQualityConvergence(1, false));
9313 factory.DepleteTaskQueues();
9314 Mock::VerifyAndClearExpectations(adapter_ptr);
9315 Mock::VerifyAndClearExpectations(&factory.GetMockFakeEncoder());
9316}
9317
Markus Handell2e0f4f02021-12-21 19:14:58 +01009318TEST(VideoStreamEncoderFrameCadenceTest,
9319 RequestsRefreshFramesWhenCadenceAdapterInstructs) {
9320 auto adapter = std::make_unique<MockFrameCadenceAdapter>();
9321 auto* adapter_ptr = adapter.get();
9322 MockVideoSourceInterface mock_source;
9323 SimpleVideoStreamEncoderFactory factory;
9324 FrameCadenceAdapterInterface::Callback* video_stream_encoder_callback =
9325 nullptr;
9326 EXPECT_CALL(*adapter_ptr, Initialize)
9327 .WillOnce(Invoke([&video_stream_encoder_callback](
9328 FrameCadenceAdapterInterface::Callback* callback) {
9329 video_stream_encoder_callback = callback;
9330 }));
9331 TaskQueueBase* encoder_queue = nullptr;
9332 auto video_stream_encoder =
9333 factory.Create(std::move(adapter), &encoder_queue);
9334 video_stream_encoder->SetSource(
9335 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9336 VideoEncoderConfig config;
9337 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9338 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9339 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9340 PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/2);
9341 // Ensure the encoder is set up.
9342 factory.DepleteTaskQueues();
9343
Markus Handell818e7fb2021-12-30 13:01:33 +01009344 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest)
9345 .WillOnce(Invoke([video_stream_encoder_callback] {
9346 video_stream_encoder_callback->RequestRefreshFrame();
9347 }));
Markus Handell2e0f4f02021-12-21 19:14:58 +01009348 EXPECT_CALL(mock_source, RequestRefreshFrame);
9349 video_stream_encoder->SendKeyFrame();
9350 factory.DepleteTaskQueues();
9351 Mock::VerifyAndClearExpectations(adapter_ptr);
9352 Mock::VerifyAndClearExpectations(&mock_source);
9353
Markus Handell818e7fb2021-12-30 13:01:33 +01009354 EXPECT_CALL(*adapter_ptr, ProcessKeyFrameRequest);
Markus Handell2e0f4f02021-12-21 19:14:58 +01009355 EXPECT_CALL(mock_source, RequestRefreshFrame).Times(0);
9356 video_stream_encoder->SendKeyFrame();
9357 factory.DepleteTaskQueues();
9358}
9359
Markus Handell818e7fb2021-12-30 13:01:33 +01009360TEST(VideoStreamEncoderFrameCadenceTest,
9361 RequestsRefreshFrameForEarlyZeroHertzKeyFrameRequest) {
9362 SimpleVideoStreamEncoderFactory factory;
9363 auto encoder_queue =
9364 factory.GetTimeController()->GetTaskQueueFactory()->CreateTaskQueue(
9365 "EncoderQueue", TaskQueueFactory::Priority::NORMAL);
9366
9367 // Enables zero-hertz mode.
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009368 test::ScopedKeyValueConfig field_trials(
9369 "WebRTC-ZeroHertzScreenshare/Enabled/");
Markus Handell818e7fb2021-12-30 13:01:33 +01009370 auto adapter = FrameCadenceAdapterInterface::Create(
Jonas Oreland8ca06132022-03-14 12:52:48 +01009371 factory.GetTimeController()->GetClock(), encoder_queue.get(),
9372 field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009373 FrameCadenceAdapterInterface* adapter_ptr = adapter.get();
9374
9375 MockVideoSourceInterface mock_source;
9376 auto video_stream_encoder = factory.CreateWithEncoderQueue(
Jonas Orelandc7f691a2022-03-09 15:12:07 +01009377 std::move(adapter), std::move(encoder_queue), &field_trials);
Markus Handell818e7fb2021-12-30 13:01:33 +01009378
9379 video_stream_encoder->SetSource(
9380 &mock_source, webrtc::DegradationPreference::MAINTAIN_FRAMERATE);
9381 VideoEncoderConfig config;
9382 config.content_type = VideoEncoderConfig::ContentType::kScreen;
9383 test::FillEncoderConfiguration(kVideoCodecVP8, 1, &config);
9384 video_stream_encoder->ConfigureEncoder(std::move(config), 0);
9385
9386 // Eventually expect a refresh frame request when requesting a key frame
9387 // before initializing zero-hertz mode. This can happen in reality because the
9388 // threads invoking key frame requests and constraints setup aren't
9389 // synchronized.
9390 EXPECT_CALL(mock_source, RequestRefreshFrame);
9391 video_stream_encoder->SendKeyFrame();
9392 adapter_ptr->OnConstraintsChanged(VideoTrackSourceConstraints{0, 30});
9393 factory.DepleteTaskQueues();
9394}
9395
perkj26091b12016-09-01 01:17:40 -07009396} // namespace webrtc